From 5e329c83eb6f581904565b958c6c0b79376e4dff Mon Sep 17 00:00:00 2001 From: Mark Bolwell Date: Wed, 29 Jul 2026 16:40:13 +0100 Subject: [PATCH 01/18] Updated core files Added pipelines Added moelcule Signed-off-by: Mark Bolwell --- .ansible-lint | 1 + .github/workflows/ci.yml | 4 +- .github/workflows/molecule.yml | 39 +++++ .gitignore | 1 + CHANGELOG.md | 138 ++++------------ DEVELOPMENT.md | 31 +++- README.md | 4 +- extensions/molecule/default/Dockerfile.j2 | 34 ++++ extensions/molecule/default/converge.yml | 94 +++++++++++ .../molecule/default/molecule-entrypoint.sh | 9 ++ extensions/molecule/default/molecule.yml | 73 +++++++++ extensions/molecule/default/prepare.yml | 99 ++++++++++++ extensions/molecule/default/verify.yml | 107 +++++++++++++ extensions/molecule/rollback/Dockerfile.j2 | 34 ++++ extensions/molecule/rollback/converge.yml | 148 ++++++++++++++++++ .../molecule/rollback/molecule-entrypoint.sh | 9 ++ extensions/molecule/rollback/molecule.yml | 73 +++++++++ extensions/molecule/rollback/prepare.yml | 90 +++++++++++ extensions/molecule/rollback/verify.yml | 87 ++++++++++ galaxy.yml | 9 ++ .../tasks/manage_crypto_policy.yml | 16 +- roles/ssh_key_rotation/tasks/verify.yml | 25 ++- 22 files changed, 994 insertions(+), 131 deletions(-) create mode 100644 .github/workflows/molecule.yml create mode 100644 extensions/molecule/default/Dockerfile.j2 create mode 100644 extensions/molecule/default/converge.yml create mode 100755 extensions/molecule/default/molecule-entrypoint.sh create mode 100644 extensions/molecule/default/molecule.yml create mode 100644 extensions/molecule/default/prepare.yml create mode 100644 extensions/molecule/default/verify.yml create mode 100644 extensions/molecule/rollback/Dockerfile.j2 create mode 100644 extensions/molecule/rollback/converge.yml create mode 100755 extensions/molecule/rollback/molecule-entrypoint.sh create mode 100644 extensions/molecule/rollback/molecule.yml create mode 100644 extensions/molecule/rollback/prepare.yml create mode 100644 extensions/molecule/rollback/verify.yml diff --git a/.ansible-lint b/.ansible-lint index c821c91..35f3eaf 100644 --- a/.ansible-lint +++ b/.ansible-lint @@ -4,6 +4,7 @@ profile: production exclude_paths: - .cache/ - .github/ + - extensions/molecule/ warn_list: - experimental diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 72d8041..5ff0f42 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,10 +12,10 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out repository - uses: actions/checkout@v4 + uses: actions/checkout@v7 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v7 with: python-version: "3.12" diff --git a/.github/workflows/molecule.yml b/.github/workflows/molecule.yml new file mode 100644 index 0000000..367511d --- /dev/null +++ b/.github/workflows/molecule.yml @@ -0,0 +1,39 @@ +--- +name: Molecule + +on: + push: + branches: [main, devel] + pull_request: + branches: [main, devel] + workflow_dispatch: + +jobs: + molecule: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + scenario: [default, rollback] + steps: + - name: Check out repository + uses: actions/checkout@v7 + + - name: Set up Python + uses: actions/setup-python@v7 + with: + python-version: "3.12" + + - name: Install Ansible, Molecule, and the docker driver + run: | + python -m pip install --upgrade pip + pip install "ansible>=2.18" "molecule>=25.0" "molecule-plugins[docker]" + + - name: Install collection dependencies + run: ansible-galaxy install -r requirements.yml + + - name: Run Molecule scenario + run: molecule test -s ${{ matrix.scenario }} + env: + PY_COLORS: "1" + ANSIBLE_FORCE_COLOR: "1" diff --git a/.gitignore b/.gitignore index bd2bffa..9b78f75 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,7 @@ venv/ *.iml .vagrant/ *.log +.mcp.* .ansible/ ansible_collections/ inventory.ini diff --git a/CHANGELOG.md b/CHANGELOG.md index d1f7b70..15b366c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,122 +1,42 @@ # Changelog -All notable changes to this project will be documented in this file. - -The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), -and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - ## [Unreleased] -### Added - -- Phase 0 now does a full pre-flight check locally, before any host is touched: it makes sure `old_private_key`, `new_private_key`, `old_public_key_file`, and `new_public_key_file` are all provided, recognizes the new key's type and size via `ssh-keygen -l` against the configurable `ssh_key_rotation_accepted_key_types`/`ssh_key_rotation_reject_key_types` allowlists (enforcing `ssh_key_rotation_min_rsa_bits` for RSA), and confirms the new private key genuinely pairs with the new public key file. -- `ssh_key_rotation_pqc_key_types`, a forward-compatible allowlist slot for post-quantum signature key types, ready for whenever OpenSSH (or an OQS-OpenSSH build) supports them. See the README's "Key Validation (PQC readiness)" section for the reasoning. -- Phase 1 now runs `sshd -T` after reloading and hard-fails the host, before Phase 2 runs, if the new key's real signature algorithm isn't in the target's effective `PubkeyAcceptedAlgorithms`. -- `Match` block detection in `sshd_config`, alongside the existing drop-in detection. -- Automatic backups of `authorized_keys` and `sshd_config` before every edit. -- Opt-in PQC/hybrid algorithm negotiation, off by default. `ssh_key_rotation_pqc_kex_algorithms`, `ssh_key_rotation_pqc_pubkey_algorithms`, and `ssh_key_rotation_pqc_ca_signature_algorithms` get appended to `KexAlgorithms`, `PubkeyAcceptedAlgorithms`/`HostKeyAlgorithms`, and `CASignatureAlgorithms` respectively, validated with `sshd -t` before writing. Phase 0 checks the control node's own ssh client actually supports what's being asked for first, and `ansible_ssh_extra_args` carries the algorithms into this playbook's own connections without touching any file on the control node. `ssh_key_rotation_manage_crypto_policy` and `ssh_key_rotation_crypto_policy_setting` can also manage RHEL/Fedora's system-wide crypto-policy on both ends. Full details in the README's "PQC Algorithm Negotiation" section. -- `ssh_key_rotation_crypto_policy_setting` now supports RHEL/Fedora's `BASE:MODULE` subpolicy syntax (e.g. `FIPS:PQ`), and before ever calling `update-crypto-policies --set` (on the control node in Phase 0, and the target in Phase 1), the playbook discovers what `*.pmod` modules actually exist under `/usr/share/crypto-policies/policies/modules/` and `/etc/crypto-policies/policies/modules/` and fails fast if a named module isn't present. This was added after finding that AlmaLinux/RHEL 9's `FIPS` policy alone has no PQC key exchange, but ships a built-in `PQ.pmod` module that adds `mlkem768x25519-sha256` when combined as `FIPS:PQ` - confirmed with `sshd -T` against a real AlmaLinux 9.8 host, before and after the switch. See the README's "Combining a base policy with a subpolicy module" section. -- **Automatic rollback in the verify stage.** Removing the old key and disabling legacy auth now run inside an Ansible `block`/`rescue`. If anything in that block fails, `rescue` immediately restores `authorized_keys` and `sshd_config` from backups taken moments earlier (on the same still-open connection, before it could be lost), reloads sshd, re-confirms connectivity, and fails with a message stating exactly what was restored and why. No separate rollback playbook or manual SSH access is needed for this failure window. -- **Improved validation checks for keys prior to running.** Phase 0 now confirms `old_private_key` and `old_public_key_file` actually exist and that the old public key parses correctly, failing fast on the control node instead of partway through a real rotation. It also warns (without blocking) if the old and new keys turn out to be identical, since that's a no-op that would otherwise only be caught much later. - ### Fixed -Found during a real rotation against a RHEL-family host running the `FIPS` crypto-policy, which locked the host out of SSH pubkey auth entirely (recovered via a VM snapshot rollback, not by this playbook): - -- **The Phase 1 "does the target accept the new key" check could silently pass when it shouldn't have.** It used to search for the key type's name (e.g. `"ed25519"`) as a substring anywhere in the full `sshd -T` output. That output also contains unrelated lines like `hostkey /etc/ssh/ssh_host_ed25519_key`, so the substring matched and suppressed the warning even though `PubkeyAcceptedAlgorithms` didn't actually include `ssh-ed25519` (the target's `FIPS` policy only allows `ecdsa-sha2-*`/`rsa-sha2-*`). Phase 1 reported success, and Phase 2 went on to remove the old key anyway. This check now extracts the `PubkeyAcceptedAlgorithms` line specifically, maps the new key's type to its real OpenSSH signature algorithm name(s) via a new `ssh_key_rotation_key_type_signature_algorithms` variable, and **fails the host** (rather than warning) if there's no overlap - before Phase 2 ever runs. -- **Phase 2's "new key works" check could be a false positive on top of SSH `ControlPersist`.** Ansible's SSH connections are commonly multiplexed by `ControlMaster`/`ControlPersist` (this repo's own `ansible.cfg` recommendation enables it), which keys the multiplexed socket by host/port/user, not by identity file. If Phase 1's connection (authenticated with the *old* key) was still open when Phase 2 started, changing `ansible_ssh_private_key_file` alone wasn't enough to guarantee a fresh handshake, later tasks could silently reuse the still-open old-key connection. Phase 2 now runs `meta: reset_connection` as its first task, before gathering facts or pinging, so the "new key works" check is always a genuine, non-multiplexed authentication attempt. -- The PQC "missing algorithm" check had the same substring-of-the-whole-dump issue as above; it now compares against the specific `KexAlgorithms`/`PubkeyAcceptedAlgorithms` lines only. +- Verify stage rollback no longer aborts if the sshd reload fails - it finishes restoring, re-confirms connectivity, and reports that sshd needs a manual reload. +- Boolean role variables are coerced with `| bool`, so passing them via `-e` (e.g. `-e ssh_key_rotation_disable_password_auth=false`) no longer fails on ansible-core 2.20. +- Molecule old-key-removal check matched base64 key material as a regex, so it could pass while the old key was still authorized. +- Molecule key-authentication checks ran over the container connection, which ignores SSH keys and so could never detect a failure; they now drive a real ssh client. +- Molecule idempotency check required `changed=0` from the timestamped backup tasks, which cannot be idempotent by design; it now allows those and fails on anything else. +- Molecule rollback check compared `authorized_keys` against a checkpoint taken before the install stage, demanding the rescue undo work it never performs. ### Changed -- `.gitignore` now excludes generated private key material (public keys are still tracked) -- **All rotation logic now lives in a proper role, `roles/ssh_key_rotation/`, instead of three standalone phase playbooks.** `playbooks/rotate.yml` is now a thin entry point that runs the role three times, once per stage (`tasks_from: validate|install|verify`), each as its own play so validate can run on `localhost`, install can connect with the OLD key, and verify can connect with the NEW key. This fixes the underlying reason the phase playbooks had to redeclare the same `vars:` and duplicate the same `Reload sshd` handler in every play: role `defaults/main.yml` and `handlers/main.yml` are shared automatically across all of a role's task files, which plain `import_playbook` plays have no equivalent for. -- **Every role-local variable now has an `ssh_key_rotation_` prefix** (e.g. `manage_crypto_policy` → `ssh_key_rotation_manage_crypto_policy`, `pqc_kex_algorithms` → `ssh_key_rotation_pqc_kex_algorithms`), to satisfy `ansible-lint`'s production-profile `var-naming[no-role-prefix]` rule and avoid colliding with other roles/host vars. **This is a breaking change** if you were passing any of these via `-e`; see the Migration Guide below. The four required variables (`old_private_key`, `new_private_key`, `old_public_key_file`, `new_public_key_file`) and `ansible_user`/`ansible_ssh_extra_args` are unaffected. -- Added `ansible.cfg` (`roles_path = ./roles`) so `playbooks/rotate.yml` can find the role when run directly from this repo, without installing it as a collection first. -- **Deduplicated the RHEL/Fedora crypto-policy logic** into a new shared `roles/ssh_key_rotation/tasks/manage_crypto_policy.yml`, included via `include_tasks` from both the validate stage (control node) and the install stage (target host) instead of maintaining two near-identical ~90-line copies. No behavior change; internal fact names used only within that file (e.g. `ssh_key_rotation_crypto_policy_current`) are no longer prefixed with `local_` on the control-node call, since each call site's facts belong to a different host and can't collide. -- **Deduplicated the Debian-vs-RHEL sshd service name lookup** (`ssh_key_rotation_sshd_service`) into a single computed default in `defaults/main.yml`, instead of an identical `set_fact` task repeated in both `install.yml` and `verify.yml`. -- Redrew the README's "Where each piece lives" PQC diagram as three separate subgraphs (labeled `validate`/`install`/`verify`, matching the actual stage names) with plain-language node labels, instead of one long flowchart full of variable/command names. The `verify` subgraph now also shows the safety gate (abort if the new key doesn't authenticate), the `authorized_keys` backup, and the old key removal, not just the final "authenticated" outcome. +- Molecule scenarios moved from `molecule/` to `extensions/molecule/`, which also corrects the relative roles path. +- Molecule test images install `openssh-server` at build time rather than during prepare. -## [1.0.0] - 2026-06-24 +## [Initial] ### Added -- Initial release of the SSH Key Rotation Ansible collection -- Two-phase SSH key rotation playbook (`playbooks/rotate.yml`): Phase 1 installs the new key and enables public key authentication, Phase 2 verifies the new key works, then removes the old key and disables legacy auth -- Safety gates to prevent accidental lockouts: the new key has to authenticate in Phase 2 before any cleanup runs, and every `sshd_config` change is validated with `sshd -t` first -- Cross-OS support for Debian/Ubuntu and RHEL/CentOS -- Detection and warnings for `/etc/ssh/sshd_config.d/` drop-in files -- Optional auth method disabling (password, keyboard-interactive) -- Zero-downtime operation, using a service reload instead of a restart -- A proper README with usage examples and a troubleshooting guide -- Collection metadata in `galaxy.yml` for Ansible Galaxy distribution - ---- - -## Planned for Future Releases - -### [1.1.0] - Planned - -- [ ] Role for key pre-flight validation (format, permissions, compatibility) -- [ ] Role for per-OS customization (selinux, firewalld, etc.) -- [ ] Module for validating `authorized_keys` integrity -- [ ] Support for key rotation with certificate-based auth -- [ ] Ansible test suite with test container targets - -### [2.0.0] - Planned - -- [ ] Support for HSM-backed keys -- [ ] Multi-key rotation in a single run -- [ ] Async key rotation for large fleets -- [ ] Integration with external secret management (HashiCorp Vault, AWS Secrets Manager) -- [ ] Comprehensive audit logging module - ---- - -## Migration Guide - -### From unprefixed variables to `ssh_key_rotation_*` (Unreleased) - -If you were passing any optional variable other than the four required key paths, `ansible_user`, or `ansible_ssh_extra_args`, add the `ssh_key_rotation_` prefix: - -| Old | New | -|-----|-----| -| `accepted_key_types` | `ssh_key_rotation_accepted_key_types` | -| `pqc_key_types` | `ssh_key_rotation_pqc_key_types` | -| `reject_key_types` | `ssh_key_rotation_reject_key_types` | -| `min_rsa_bits` | `ssh_key_rotation_min_rsa_bits` | -| `pqc_kex_algorithms` | `ssh_key_rotation_pqc_kex_algorithms` | -| `pqc_pubkey_algorithms` | `ssh_key_rotation_pqc_pubkey_algorithms` | -| `pqc_ca_signature_algorithms` | `ssh_key_rotation_pqc_ca_signature_algorithms` | -| `manage_crypto_policy` | `ssh_key_rotation_manage_crypto_policy` | -| `crypto_policy_setting` | `ssh_key_rotation_crypto_policy_setting` | -| `crypto_policy_add_modules` | `ssh_key_rotation_crypto_policy_add_modules` | -| `disable_password_auth` | `ssh_key_rotation_disable_password_auth` | -| `disable_kbd_interactive` | `ssh_key_rotation_disable_kbd_interactive` | -| `make_exclusive` | `ssh_key_rotation_make_exclusive` | - -### From standalone playbook to collection - -If you were previously running this as a standalone `ssh_key_rotation.yml` playbook: - -**Old usage:** -```bash -ansible-playbook ssh_key_rotation.yml -i inventory.ini -e ... -``` - -**New usage:** -```bash -ansible-playbook krameff.ssh_key_rotation.rotate -i inventory.ini -e ... -``` - -**Steps:** - -1. Install the collection: `ansible-galaxy collection install krameff.ssh_key_rotation` -2. Update your playbook references from `ssh_key_rotation.yml` to `krameff.ssh_key_rotation.rotate` -3. All variables and behavior remain identical - ---- - -## Version History - -- **1.0.0** (2026-06-24): Initial collection release from standalone playbook +- Two-phase key rotation via a proper role: install the new key, verify it works, then remove the old key and disable legacy auth. +- Cross-OS support for Debian/Ubuntu and RHEL/CentOS, zero-downtime (reload not restart). +- Pre-flight validation of key paths, new key type/size, and key pair match, before touching any host. +- `ssh_key_rotation_pqc_key_types` allowlist slot, reserved for future PQC key types. +- Install stage now hard-fails if `sshd -T` won't actually accept the new key's algorithm. +- `Match` block detection in `sshd_config`. +- Automatic backup of `authorized_keys` and `sshd_config` before every edit. +- Opt-in PQC/hybrid algorithm negotiation (off by default), plus RHEL/Fedora crypto-policy management. See README "PQC Algorithm Negotiation". +- Crypto-policy support for RHEL/Fedora's `BASE:MODULE` syntax (e.g. `FIPS:PQ`), with a check that the module exists first. +- Automatic rollback in the verify stage if old-key removal or auth cleanup fails. +- Stricter old-key validation (files exist, key parses); warns if old and new keys are identical. +- New-key acceptance check used a substring match against `sshd -T` output, which could false-positive on unrelated lines - the old key got removed even though the new algorithm wasn't actually accepted. Now parses `PubkeyAcceptedAlgorithms` directly. +- New-key auth check could reuse a stale multiplexed SSH connection from the old key, making the check meaningless. Now forces `meta: reset_connection` first. +- Same substring bug in the PQC "missing algorithm" check - fixed the same way. +- `.gitignore` excludes generated private keys. +- Rotation logic moved into a proper role (`roles/ssh_key_rotation/`) instead of three standalone playbooks. +- All role variables now use an `ssh_key_rotation_` prefix (e.g. `manage_crypto_policy` → `ssh_key_rotation_manage_crypto_policy`). +- Added `ansible.cfg` so the role is found locally without installing as a collection. +- Deduplicated RHEL/Fedora crypto-policy logic and the Debian/RHEL sshd service lookup. +- Redrew the README's PQC diagram to match actual stage names and show the rollback gate. diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 7080b25..011796d 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -16,8 +16,13 @@ krameff-ssh_key_rotation/ ├── .ansible-lint # ansible-lint configuration (production profile) ├── .github/ │ └── workflows/ -│ └── ci.yml # Lint + syntax-check on push/PR +│ ├── ci.yml # Lint + syntax-check on push/PR +│ └── molecule.yml # Functional Molecule/Docker tests (default + rollback scenarios) on push/PR ├── ansible.cfg # Sets roles_path so playbooks/rotate.yml finds roles/ without installing the collection +├── extensions/ +│ └── molecule/ # Molecule scenarios (the location current Molecule expects) +│ ├── default/ # Full rotation over real SSH, twice, to also prove idempotency +│ └── rollback/ # Breaks a rotation mid-verify to prove the rescue block restores access ├── meta/ │ └── runtime.yml # Ansible version requirements ├── playbooks/ @@ -128,16 +133,30 @@ ansible-playbook -i inventory.ini playbooks/rotate.yml --check ansible-lint ``` -### Integration Testing with Molecule (Future) +### Integration Testing with Molecule + +Two scenarios live under `extensions/molecule/` and run the real `playbooks/rotate.yml` (or, for +the rollback scenario, the role's stages directly) against live Docker containers +(Ubuntu 22.04, Rocky Linux 9, Rocky Linux 10) with sshd installed and running: ```bash -# Create a scenario -molecule init scenario -d -s +pip install "molecule>=25.0" "molecule-plugins[docker]" + +# Full rotation + idempotency check +molecule test -s default -# Test the scenario -molecule test +# Deliberately breaks the second rotation mid-verify to prove the block/rescue +# in roles/ssh_key_rotation/tasks/verify.yml actually restores access +molecule test -s rollback ``` +Both scenarios generate their own ephemeral SSH keypairs and inventory into Molecule's +per-run ephemeral directory - they do not use the root-level `test_vars.yml` or +`test_rsa*`/`test_ecdsa*`/`test_ed25519` files. Those root-level files are for local, +manual testing only (e.g. `ansible-playbook -i inventory.ini playbooks/rotate.yml -e +@test_vars.yml`); edit `test_vars.yml`'s paths for your own machine before using it, since +the checked-in paths are just examples. + ## Building and Publishing ### Build a Distribution Tarball diff --git a/README.md b/README.md index 2bf4e23..465f1fb 100644 --- a/README.md +++ b/README.md @@ -269,7 +269,7 @@ The whole point of this playbook is that you should never be able to lock yourse - Every `sshd_config` change is validated with `sshd -t` before it's written. - Configuration is applied with a reload, never a restart, so sessions already open stay open. - `sshd_config` and `authorized_keys` are both backed up before every edit, so you can always roll back by hand. -- Phase 2's cleanup (removing the old key, disabling legacy auth) runs inside an Ansible `block`/`rescue`. If anything in it fails partway through, `rescue` automatically restores `authorized_keys` and `sshd_config` from the backups just taken, reloads sshd, re-confirms connectivity, and fails with a clear message - all on the same still-open connection, before it could be lost. +- Phase 2's cleanup (removing the old key, disabling legacy auth) runs inside an Ansible `block`/`rescue`. If anything in it fails partway through, `rescue` automatically restores `authorized_keys` and `sshd_config` from the backups just taken, reloads sshd, re-confirms connectivity, and fails with a clear message - all on the same still-open connection, before it could be lost. A reload that itself fails does not abort the rollback: the restored files are already on disk and sshd reads `authorized_keys` per connection, so the rollback finishes and the failure message tells you to reload sshd by hand. ### Phase 0: validate locally @@ -296,7 +296,7 @@ The whole point of this playbook is that you should never be able to lock yourse 2. Reconnect and gather facts with the new private key - this is the authentication gate 3. Ping the host to confirm the new key works 4. Assert the new key actually authenticated before doing anything else -5. From here on, steps 6-10 run inside a `block`/`rescue`: if any of them fail, `rescue` automatically restores `authorized_keys` and `sshd_config` from the backups taken below, reloads sshd, re-confirms connectivity, and fails with a clear message instead of leaving the host half-changed +5. From here on, steps 6-10 run inside a `block`/`rescue`: if any of them fail, `rescue` automatically restores `authorized_keys` and `sshd_config` from the backups taken below, reloads sshd (best effort - a failed reload is reported, not fatal, since the restored files are already in place), re-confirms connectivity, and fails with a clear message instead of leaving the host half-changed 6. Back up `authorized_keys`, then remove the old public key from it 7. Optionally make `authorized_keys` exclusive to the new key 8. Disable legacy auth methods (password, keyboard-interactive) if requested, backing up `sshd_config` first diff --git a/extensions/molecule/default/Dockerfile.j2 b/extensions/molecule/default/Dockerfile.j2 new file mode 100644 index 0000000..9bfa74d --- /dev/null +++ b/extensions/molecule/default/Dockerfile.j2 @@ -0,0 +1,34 @@ +# Molecule managed +# +# Wraps the upstream geerlingguy image (already has python/ansible prereqs) with an +# entrypoint that pre-creates root's ansible remote_tmp dir before systemd starts, so the +# first ansible task never races container/systemd boot for that mkdir. +FROM {{ item.image }} + +# These test images don't ship openssh-server. Install it at build time (not via an Ansible +# task in prepare.yml) so any slowness/hang shows up in the docker build log instead of being +# buried in Ansible task output. +{% if 'ubuntu' in item.image or 'debian' in item.image %} +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends openssh-server \ + && rm -rf /var/lib/apt/lists/* +{% else %} +RUN dnf install -y openssh-server \ + && dnf clean all + +# rockylinux9 ships /etc/shadow as mode 0000 (the RHEL default) but with a pam old enough +# (1.5.1-28.el9) that pam_unix's ACCOUNT phase resolves shadow via the unix_chkpwd helper. +# That helper links libcap-ng and deliberately drops its capabilities, so it loses +# DAC_OVERRIDE and cannot open a 0000 file even as uid 0 - the lookup fails with +# PAM_AUTHINFO_UNAVAIL. sshd (UsePAM yes) then rejects every login with "Access denied for +# user root by PAM account configuration", which the client sees as a bare +# "Connection closed by port 22"; sudo fails the same way. rockylinux10's pam 1.6.1 +# does not use the helper here, which is why only rockylinux9 is affected. +# 0400 root:root is still root-only, so this grants no access that 0000 didn't already imply. +RUN chmod 0400 /etc/shadow +{% endif %} + +COPY molecule-entrypoint.sh /usr/local/bin/molecule-entrypoint.sh +RUN chmod +x /usr/local/bin/molecule-entrypoint.sh + +ENTRYPOINT ["/usr/local/bin/molecule-entrypoint.sh"] diff --git a/extensions/molecule/default/converge.yml b/extensions/molecule/default/converge.yml new file mode 100644 index 0000000..2faa06d --- /dev/null +++ b/extensions/molecule/default/converge.yml @@ -0,0 +1,94 @@ +--- +# Runs the collection's real entry point (playbooks/rotate.yml, all 3 stages) against the +# generated inventory/vars from prepare.yml -- this is what proves the shipped playbook works +# end to end, not just the role in isolation. Then runs it again, unchanged, to prove +# idempotency: a second identical rotation should report changed=0 across the board. + +- name: Converge | Run the real rotation playbook end to end (first pass) + hosts: localhost + connection: local + gather_facts: false + vars: + ssh_key_rotation_test_dir: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}" + ssh_key_rotation_repo_root: "{{ playbook_dir }}/../../.." + tasks: + - name: Converge | First rotation run + ansible.builtin.command: >- + ansible-playbook -i {{ ssh_key_rotation_test_dir }}/rotation_inventory.ini + {{ ssh_key_rotation_repo_root }}/playbooks/rotate.yml + -e @{{ ssh_key_rotation_test_dir }}/rotation_vars.yml + environment: + ANSIBLE_ROLES_PATH: "{{ ssh_key_rotation_repo_root }}/roles" + ANSIBLE_HOST_KEY_CHECKING: "false" + changed_when: true + register: ssh_key_rotation_first_run + + - name: Converge | Show first run output + ansible.builtin.debug: + var: ssh_key_rotation_first_run.stdout_lines + + - name: Converge | Second (idempotency) rotation run with identical vars + ansible.builtin.command: >- + ansible-playbook -i {{ ssh_key_rotation_test_dir }}/rotation_inventory.ini + {{ ssh_key_rotation_repo_root }}/playbooks/rotate.yml + -e @{{ ssh_key_rotation_test_dir }}/rotation_vars.yml + environment: + ANSIBLE_ROLES_PATH: "{{ ssh_key_rotation_repo_root }}/roles" + ANSIBLE_HOST_KEY_CHECKING: "false" + # JSON callback so idempotency can be judged per task below, rather than from the PLAY + # RECAP's per-host counts (which can't say WHICH task changed). Warnings still go to + # stderr, so stdout stays valid JSON. + ANSIBLE_STDOUT_CALLBACK: json + changed_when: true + register: ssh_key_rotation_second_run + + # The three "Back up ..." tasks write to a dest containing ansible_date_time, so they produce a + # new file on every run and can never report changed=0. That's deliberate: each run must + # snapshot the state before that run, and reusing a fixed filename would overwrite the original + # pre-rotation backup. So they are expected to change, and anything else changing is a real + # idempotency regression - which is what this splits apart. + - name: Converge | Split the second run's changed tasks into expected backups vs unexpected + vars: + ssh_key_rotation_second_run_json: "{{ ssh_key_rotation_second_run.stdout | from_json }}" + ansible.builtin.set_fact: + ssh_key_rotation_unexpected_changed: >- + {%- set found = [] -%} + {%- for play in ssh_key_rotation_second_run_json.plays -%} + {%- for task in play.tasks -%} + {%- for host, result in task.hosts.items() -%} + {%- if (result.changed | default(false)) and 'Back up' not in task.task.name -%} + {%- set _ = found.append(task.task.name ~ ' on ' ~ host) -%} + {%- endif -%} + {%- endfor -%} + {%- endfor -%} + {%- endfor -%} + {{ found | unique | list }} + ssh_key_rotation_expected_backup_changed: >- + {%- set found = [] -%} + {%- for play in ssh_key_rotation_second_run_json.plays -%} + {%- for task in play.tasks -%} + {%- for host, result in task.hosts.items() -%} + {%- if (result.changed | default(false)) and 'Back up' in task.task.name -%} + {%- set _ = found.append(task.task.name) -%} + {%- endif -%} + {%- endfor -%} + {%- endfor -%} + {%- endfor -%} + {{ found | unique | list }} + + - name: Converge | Show what the second (idempotency) run changed + ansible.builtin.debug: + msg: + expected_timestamped_backups: "{{ ssh_key_rotation_expected_backup_changed }}" + unexpected_changes: "{{ ssh_key_rotation_unexpected_changed }}" + + - name: Converge | Assert the second run changed nothing beyond the timestamped backups + ansible.builtin.assert: + that: ssh_key_rotation_unexpected_changed | length == 0 + fail_msg: >- + Second identical rotation run changed task(s) other than the timestamped backups, so + rotation is not idempotent: {{ ssh_key_rotation_unexpected_changed }} + success_msg: >- + Second identical rotation run was idempotent: the only changes were the + {{ ssh_key_rotation_expected_backup_changed | length }} timestamped backup file(s), + which are re-created on every run by design. diff --git a/extensions/molecule/default/molecule-entrypoint.sh b/extensions/molecule/default/molecule-entrypoint.sh new file mode 100755 index 0000000..d0a681e --- /dev/null +++ b/extensions/molecule/default/molecule-entrypoint.sh @@ -0,0 +1,9 @@ +#!/bin/bash +set -euo pipefail + +# Pre-create root's ansible remote_tmp dir before handing off to the real container command +# (systemd via /usr/sbin/init). This runs synchronously before the container is reported as +# started, so it can never race the first ansible connection's own mkdir of this same path. +install -d -m 0700 /root/.ansible/tmp + +exec "$@" diff --git a/extensions/molecule/default/molecule.yml b/extensions/molecule/default/molecule.yml new file mode 100644 index 0000000..11793f4 --- /dev/null +++ b/extensions/molecule/default/molecule.yml @@ -0,0 +1,73 @@ +--- +driver: + name: docker +platforms: + - name: ssh-key-rotation-ubuntu2204 + image: geerlingguy/docker-ubuntu2204-ansible:latest + command: /usr/sbin/init + privileged: true + cgroupns_mode: host + pre_build_image: false + tmpfs: + - /run + - /run/lock + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:rw + groups: + - rotate + - name: ssh-key-rotation-rockylinux9 + image: geerlingguy/docker-rockylinux9-ansible:latest + command: /usr/sbin/init + privileged: true + cgroupns_mode: host + pre_build_image: false + tmpfs: + - /run + - /run/lock + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:rw + groups: + - rotate + - name: ssh-key-rotation-rockylinux10 + image: geerlingguy/docker-rockylinux10-ansible:latest + command: /usr/sbin/init + privileged: true + cgroupns_mode: host + pre_build_image: false + tmpfs: + - /run + - /run/lock + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:rw + groups: + - rotate +provisioner: + name: ansible + env: + ANSIBLE_ROLES_PATH: "../../../roles" + inventory: + group_vars: + # The docker connection plugin doesn't invoke a login shell, so `~` in the default + # remote_tmp path doesn't reliably expand for the "rotate" containers. Pin it to the + # absolute path molecule-entrypoint.sh already pre-creates for root before systemd + # starts. Scoped to this group only - localhost (used by molecule's own internal + # playbooks and the key-generation play) must keep its normal, non-root remote_tmp. + rotate: + ansible_remote_tmp: /root/.ansible/tmp + # These containers are entered as root. Saying so explicitly means Ansible skips + # become for the role's `become: true` tasks (become_user root == remote_user root) + # instead of shelling out to sudo. Without this, sudo runs and fails on images whose + # pam/shadow setup can't serve an account lookup inside a container (seen on + # rockylinux9: "PAM account management error: Authentication service cannot retrieve + # authentication info"), which has nothing to do with what these tests are proving. + ansible_user: root +scenario: + test_sequence: + - syntax + - create + - prepare + - converge + - verify + - destroy +verifier: + name: ansible diff --git a/extensions/molecule/default/prepare.yml b/extensions/molecule/default/prepare.yml new file mode 100644 index 0000000..a6ed81d --- /dev/null +++ b/extensions/molecule/default/prepare.yml @@ -0,0 +1,99 @@ +--- +# Prepares each Molecule-managed container as a realistic rotation target: an ephemeral +# "old" keypair pre-authorized (simulating existing access before install.yml ever runs), +# an ephemeral "new" keypair generated but NOT yet authorized, and a generated inventory + +# vars file that converge.yml uses to invoke the real playbooks/rotate.yml over actual SSH +# (not the Molecule docker connection plugin, since that's not what the role uses in production). + +- name: Prepare | Generate ephemeral keys and the test inventory on the control node + hosts: localhost + connection: local + gather_facts: false + vars: + ssh_key_rotation_test_dir: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}" + tasks: + - name: Prepare | Generate the ephemeral OLD (RSA) keypair + ansible.builtin.command: "ssh-keygen -t rsa -b 3072 -N '' -f {{ ssh_key_rotation_test_dir }}/old_key" + args: + creates: "{{ ssh_key_rotation_test_dir }}/old_key" + + - name: Prepare | Generate the ephemeral NEW (ED25519) keypair + ansible.builtin.command: "ssh-keygen -t ed25519 -N '' -f {{ ssh_key_rotation_test_dir }}/new_key" + args: + creates: "{{ ssh_key_rotation_test_dir }}/new_key" + + - name: Prepare | Write the generated rotation vars file used by converge.yml + ansible.builtin.copy: + dest: "{{ ssh_key_rotation_test_dir }}/rotation_vars.yml" + mode: "0600" + content: | + --- + old_private_key: "{{ ssh_key_rotation_test_dir }}/old_key" + old_public_key_file: "{{ ssh_key_rotation_test_dir }}/old_key.pub" + new_private_key: "{{ ssh_key_rotation_test_dir }}/new_key" + new_public_key_file: "{{ ssh_key_rotation_test_dir }}/new_key.pub" + ssh_key_rotation_target_user: root + +- name: Prepare | Seed the OLD key into each container's authorized_keys (simulates pre-existing access) + hosts: rotate + gather_facts: true + vars: + ssh_key_rotation_test_dir: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}" + # openssh-server is installed at Dockerfile build time (see Dockerfile.j2), not here. The + # service/unit name still differs by family: Debian/Ubuntu: ssh, RedHat/Rocky: sshd. + ssh_key_rotation_sshd_service: "{{ 'sshd' if ansible_facts['os_family'] == 'RedHat' else 'ssh' }}" + tasks: + - name: Prepare | Ensure sshd is running + ansible.builtin.systemd: + name: "{{ ssh_key_rotation_sshd_service }}" + state: started + enabled: true + + - name: Prepare | Ensure PermitRootLogin allows key auth (test images default to prohibit-password already) + ansible.builtin.lineinfile: + path: /etc/ssh/sshd_config + regexp: '^#?\s*PermitRootLogin' + line: 'PermitRootLogin prohibit-password' + notify: Restart sshd for prepare + + - name: Prepare | Install the ephemeral OLD public key as pre-existing authorized access + ansible.posix.authorized_key: + user: root + key: "{{ lookup('file', ssh_key_rotation_test_dir + '/old_key.pub') }}" + state: present + + - name: Prepare | Discover this container's IP address on the docker network + ansible.builtin.command: hostname -I + changed_when: false + register: ssh_key_rotation_container_ip + + - name: Prepare | Record this container's IP for the generated inventory + ansible.builtin.set_fact: + ssh_key_rotation_container_ip: "{{ ssh_key_rotation_container_ip.stdout.split() | first }}" + + - name: Prepare | Append this host to the generated rotation inventory + ansible.builtin.lineinfile: + path: "{{ ssh_key_rotation_test_dir }}/rotation_inventory.ini" + create: true + mode: "0600" + line: >- + {{ inventory_hostname }} ansible_host={{ ssh_key_rotation_container_ip }} ansible_user=root + ansible_ssh_common_args='-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null' + insertafter: EOF + delegate_to: localhost + + - name: Prepare | Ensure the [rotate] group header is present in the generated inventory + ansible.builtin.lineinfile: + path: "{{ ssh_key_rotation_test_dir }}/rotation_inventory.ini" + line: "[rotate]" + insertbefore: BOF + create: true + mode: "0600" + delegate_to: localhost + run_once: true + + handlers: + - name: Restart sshd for prepare + ansible.builtin.systemd: + name: "{{ ssh_key_rotation_sshd_service }}" + state: restarted diff --git a/extensions/molecule/default/verify.yml b/extensions/molecule/default/verify.yml new file mode 100644 index 0000000..7e8c5ba --- /dev/null +++ b/extensions/molecule/default/verify.yml @@ -0,0 +1,107 @@ +--- +# Confirms the rotation actually did what it claims: the new key authenticates, the old key +# is rejected, authorized_keys/sshd_config reflect the role's (default) settings. + +- name: Verify | Confirm rotation results via the NEW key + hosts: rotate + gather_facts: false + vars: + ssh_key_rotation_test_dir: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}" + ansible_ssh_private_key_file: "{{ ssh_key_rotation_test_dir }}/new_key" + tasks: + - name: Verify | New key authenticates + ansible.builtin.ping: + + - name: Verify | Read authorized_keys + ansible.builtin.slurp: + src: /root/.ssh/authorized_keys + register: ssh_key_rotation_authorized_keys_content + + # Literal `in`, never `is search`: `search` treats its argument as a REGEX, and a base64 key + # blob routinely contains `+` (a quantifier), so the pattern silently fails to match a key that + # IS present. On the negative assertion below that failure is dangerous rather than merely + # flaky - `not (no match)` is True, so it would report the old key as removed while the old key + # was still authorized. + - name: Verify | New key is present in authorized_keys + ansible.builtin.assert: + that: >- + lookup('file', ssh_key_rotation_test_dir + '/new_key.pub').split()[1] + in (ssh_key_rotation_authorized_keys_content.content | b64decode) + fail_msg: "The new key is not in authorized_keys after rotation." + success_msg: "New key is present in authorized_keys." + + - name: Verify | Old key has been removed from authorized_keys + ansible.builtin.assert: + that: >- + lookup('file', ssh_key_rotation_test_dir + '/old_key.pub').split()[1] + not in (ssh_key_rotation_authorized_keys_content.content | b64decode) + fail_msg: "The old key is STILL in authorized_keys after rotation - it should have been removed." + success_msg: "Old key has been removed from authorized_keys." + + - name: Verify | Check effective sshd configuration + ansible.builtin.command: sshd -T + become: true + changed_when: false + register: ssh_key_rotation_sshd_effective + + - name: Verify | sshd_config reflects the role's default safety settings + ansible.builtin.assert: + that: + - "'pubkeyauthentication yes' in ssh_key_rotation_sshd_effective.stdout | lower" + - "'passwordauthentication no' in ssh_key_rotation_sshd_effective.stdout | lower" + - "'kbdinteractiveauthentication no' in ssh_key_rotation_sshd_effective.stdout | lower" + +# This has to drive a real ssh client, not ansible's ping. Molecule's docker driver reaches the +# rotate group over ansible_connection=community.docker.docker, which ignores +# ansible_ssh_private_key_file completely - a ping there returns "pong" even when handed a +# nonexistent key file, so it can never demonstrate that sshd rejected the old key. The checks +# below run ssh from the control node against the container's own IP (the same path converge uses), +# with IdentitiesOnly=yes so ssh cannot quietly fall back to some other identity and pass. +- name: Verify | Confirm the OLD key is now rejected over a real SSH connection + hosts: rotate + gather_facts: false + vars: + ssh_key_rotation_test_dir: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}" + ssh_key_rotation_verify_ssh_opts: >- + -o BatchMode=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null + -o IdentitiesOnly=yes -o ConnectTimeout=15 + tasks: + - name: Verify | Discover this container's IP address for the real SSH checks + ansible.builtin.command: hostname -I + changed_when: false + register: ssh_key_rotation_verify_container_ip + + # Positive control, and it runs first on purpose: without it, a host where sshd rejects + # EVERY key (wiped authorized_keys, broken sshd) would satisfy the old-key assertion below + # and pass while access had actually been lost. + - name: Verify | NEW key authenticates over real SSH (positive control) + ansible.builtin.command: >- + ssh -i {{ ssh_key_rotation_test_dir }}/new_key {{ ssh_key_rotation_verify_ssh_opts }} + root@{{ ssh_key_rotation_verify_container_ip.stdout.split() | first }} true + delegate_to: localhost + changed_when: false + + - name: Verify | Attempt a real SSH connection with the OLD key (must be refused) + ansible.builtin.command: >- + ssh -i {{ ssh_key_rotation_test_dir }}/old_key {{ ssh_key_rotation_verify_ssh_opts }} + root@{{ ssh_key_rotation_verify_container_ip.stdout.split() | first }} true + delegate_to: localhost + changed_when: false + failed_when: false + register: ssh_key_rotation_old_key_ssh + + # Both conditions matter: a non-zero rc alone would also be satisfied by a timeout or a + # refused connection, which would hide a broken test environment behind a passing assertion. + # Requiring sshd's own "Permission denied" proves the daemon answered and rejected the key. + - name: Verify | Assert sshd refused the OLD key + ansible.builtin.assert: + that: + - ssh_key_rotation_old_key_ssh.rc != 0 + - "'permission denied' in (ssh_key_rotation_old_key_ssh.stderr | default('') | lower)" + fail_msg: >- + Old key still authenticates after rotation - it should have been removed. + ssh exited {{ ssh_key_rotation_old_key_ssh.rc }}; + stderr: {{ ssh_key_rotation_old_key_ssh.stderr | default('(none)') }} + success_msg: >- + sshd refused the old key with "Permission denied", while the new key still + authenticates over the same SSH path - rotation removed old access as intended. diff --git a/extensions/molecule/rollback/Dockerfile.j2 b/extensions/molecule/rollback/Dockerfile.j2 new file mode 100644 index 0000000..9bfa74d --- /dev/null +++ b/extensions/molecule/rollback/Dockerfile.j2 @@ -0,0 +1,34 @@ +# Molecule managed +# +# Wraps the upstream geerlingguy image (already has python/ansible prereqs) with an +# entrypoint that pre-creates root's ansible remote_tmp dir before systemd starts, so the +# first ansible task never races container/systemd boot for that mkdir. +FROM {{ item.image }} + +# These test images don't ship openssh-server. Install it at build time (not via an Ansible +# task in prepare.yml) so any slowness/hang shows up in the docker build log instead of being +# buried in Ansible task output. +{% if 'ubuntu' in item.image or 'debian' in item.image %} +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends openssh-server \ + && rm -rf /var/lib/apt/lists/* +{% else %} +RUN dnf install -y openssh-server \ + && dnf clean all + +# rockylinux9 ships /etc/shadow as mode 0000 (the RHEL default) but with a pam old enough +# (1.5.1-28.el9) that pam_unix's ACCOUNT phase resolves shadow via the unix_chkpwd helper. +# That helper links libcap-ng and deliberately drops its capabilities, so it loses +# DAC_OVERRIDE and cannot open a 0000 file even as uid 0 - the lookup fails with +# PAM_AUTHINFO_UNAVAIL. sshd (UsePAM yes) then rejects every login with "Access denied for +# user root by PAM account configuration", which the client sees as a bare +# "Connection closed by port 22"; sudo fails the same way. rockylinux10's pam 1.6.1 +# does not use the helper here, which is why only rockylinux9 is affected. +# 0400 root:root is still root-only, so this grants no access that 0000 didn't already imply. +RUN chmod 0400 /etc/shadow +{% endif %} + +COPY molecule-entrypoint.sh /usr/local/bin/molecule-entrypoint.sh +RUN chmod +x /usr/local/bin/molecule-entrypoint.sh + +ENTRYPOINT ["/usr/local/bin/molecule-entrypoint.sh"] diff --git a/extensions/molecule/rollback/converge.yml b/extensions/molecule/rollback/converge.yml new file mode 100644 index 0000000..1a74fe6 --- /dev/null +++ b/extensions/molecule/rollback/converge.yml @@ -0,0 +1,148 @@ +--- +# Runs TWO rotations against the container(s): +# 1. seed_key -> key_a: a normal, fully successful rotation (this scenario leaves password +# and keyboard-interactive auth ENABLED on purpose, so run 2 below has something new to +# disable and therefore something that can fail). +# 2. key_a -> key_b: install succeeds normally, but the verify stage is given a bogus +# ssh_key_rotation_sshd_service value. That value is only exercised once verify's block +# tries to reload sshd after disabling password/keyboard-interactive auth for the first +# time (a real, first-time sshd_config change) - by then, authorized_keys has already +# been backed up and had key_a removed, so this is a legitimate way to make the rescue: +# block in roles/ssh_key_rotation/tasks/verify.yml fire, without editing any role logic. +# +# This scenario invokes the role's stages directly (like playbooks/rotate.yml does) instead of +# shelling out to that playbook, because it needs a different ssh_key_rotation_sshd_service +# value on the second rotation's verify stage only - not achievable with a single shared +# extra-vars file passed to one ansible-playbook invocation. + +- name: Converge | Validate stage (rotation 1, seed_key -> key_a) + hosts: localhost + connection: local + gather_facts: false + vars: + ssh_key_rotation_test_dir: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}" + old_private_key: "{{ ssh_key_rotation_test_dir }}/seed_key" + old_public_key_file: "{{ ssh_key_rotation_test_dir }}/seed_key.pub" + new_private_key: "{{ ssh_key_rotation_test_dir }}/key_a" + new_public_key_file: "{{ ssh_key_rotation_test_dir }}/key_a.pub" + tasks: + - name: Converge | Run role validate stage + ansible.builtin.include_role: + name: ssh_key_rotation + tasks_from: validate + +- name: Converge | Install stage (rotation 1, seed_key -> key_a) + hosts: rotate + gather_facts: true + vars: + ssh_key_rotation_test_dir: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}" + old_private_key: "{{ ssh_key_rotation_test_dir }}/seed_key" + old_public_key_file: "{{ ssh_key_rotation_test_dir }}/seed_key.pub" + new_private_key: "{{ ssh_key_rotation_test_dir }}/key_a" + new_public_key_file: "{{ ssh_key_rotation_test_dir }}/key_a.pub" + ansible_ssh_private_key_file: "{{ old_private_key }}" + ssh_key_rotation_target_user: root + tasks: + - name: Converge | Run role install stage + ansible.builtin.include_role: + name: ssh_key_rotation + tasks_from: install + +- name: Converge | Verify stage (rotation 1, seed_key -> key_a) - leave password/kbd auth ENABLED + hosts: rotate + gather_facts: false + vars: + ssh_key_rotation_test_dir: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}" + old_public_key_file: "{{ ssh_key_rotation_test_dir }}/seed_key.pub" + new_private_key: "{{ ssh_key_rotation_test_dir }}/key_a" + new_public_key_file: "{{ ssh_key_rotation_test_dir }}/key_a.pub" + ansible_ssh_private_key_file: "{{ new_private_key }}" + ssh_key_rotation_target_user: root + ssh_key_rotation_disable_password_auth: false + ssh_key_rotation_disable_kbd_interactive: false + tasks: + - name: Converge | Run role verify stage + ansible.builtin.include_role: + name: ssh_key_rotation + tasks_from: verify + +- name: Converge | Snapshot the known-good state after rotation 1 (the rollback checkpoint) + hosts: rotate + gather_facts: false + vars: + ssh_key_rotation_test_dir: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}" + ansible_ssh_private_key_file: "{{ ssh_key_rotation_test_dir }}/key_a" + tasks: + - name: Converge | Read authorized_keys after rotation 1 + ansible.builtin.slurp: + src: /root/.ssh/authorized_keys + register: ssh_key_rotation_checkpoint_authorized_keys + + - name: Converge | Save the authorized_keys checkpoint locally + ansible.builtin.copy: + content: "{{ ssh_key_rotation_checkpoint_authorized_keys.content | b64decode }}" + dest: "{{ ssh_key_rotation_test_dir }}/checkpoint_authorized_keys.{{ inventory_hostname }}" + mode: "0600" + delegate_to: localhost + +- name: Converge | Install stage (rotation 2, key_a -> key_b) + hosts: rotate + gather_facts: true + vars: + ssh_key_rotation_test_dir: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}" + old_private_key: "{{ ssh_key_rotation_test_dir }}/key_a" + old_public_key_file: "{{ ssh_key_rotation_test_dir }}/key_a.pub" + new_private_key: "{{ ssh_key_rotation_test_dir }}/key_b" + new_public_key_file: "{{ ssh_key_rotation_test_dir }}/key_b.pub" + ansible_ssh_private_key_file: "{{ old_private_key }}" + ssh_key_rotation_target_user: root + tasks: + - name: Converge | Run role install stage + ansible.builtin.include_role: + name: ssh_key_rotation + tasks_from: install + +- name: Converge | Verify stage (rotation 2, key_a -> key_b) - deliberately broken reload + hosts: rotate + gather_facts: false + vars: + ssh_key_rotation_test_dir: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}" + old_public_key_file: "{{ ssh_key_rotation_test_dir }}/key_a.pub" + new_private_key: "{{ ssh_key_rotation_test_dir }}/key_b" + new_public_key_file: "{{ ssh_key_rotation_test_dir }}/key_b.pub" + ansible_ssh_private_key_file: "{{ new_private_key }}" + ssh_key_rotation_target_user: root + ssh_key_rotation_sshd_service: "bogus-sshd-molecule-test" + tasks: + # block/rescue, not `ignore_errors` + `register` on the include_role: ignore_errors does NOT + # propagate into the tasks of an included role, so a failure inside verify.yml fails the play + # outright and any assert placed after the include never runs (the scenario could never exit 0). + # Wrapping it means the role's failure lands in this rescue, where it can be asserted on. + - name: Converge | Expect the verify stage to fail and trigger the role's rescue block + block: + - name: Converge | Run role verify stage (expected to fail and trigger the rescue block) + ansible.builtin.include_role: + name: ssh_key_rotation + tasks_from: verify + + - name: Converge | Fail if the deliberately-broken verify stage unexpectedly succeeded + ansible.builtin.fail: + msg: >- + Expected rotation 2's verify stage to fail (bogus ssh_key_rotation_sshd_service), + proving the rescue block fires - it succeeded instead, so this test proves nothing. + rescue: + # Asserting on the rollback's own diagnostic, not merely "something failed": that message is + # only ever reached if the rescue block ran all the way through (restore -> reload -> + # connectivity check -> report). A bare `is failed` would also pass when the rescue aborted + # partway, which is exactly the bug this scenario exists to catch. + - name: Converge | Assert the rescue block ran to completion and reported the rollback + ansible.builtin.assert: + that: >- + 'automatically rolled back' in (ansible_failed_result.msg | default('')) + fail_msg: >- + Rotation 2's verify stage failed, but not at the rollback diagnostic - the rescue + block did not run to completion, so access was not verified after the restore. + Last error was: {{ ansible_failed_result.msg | default('none') }} + success_msg: >- + Rescue block ran to completion: authorized_keys/sshd_config restored, connectivity + re-confirmed with the new key, and the rollback reported to the operator. diff --git a/extensions/molecule/rollback/molecule-entrypoint.sh b/extensions/molecule/rollback/molecule-entrypoint.sh new file mode 100755 index 0000000..d0a681e --- /dev/null +++ b/extensions/molecule/rollback/molecule-entrypoint.sh @@ -0,0 +1,9 @@ +#!/bin/bash +set -euo pipefail + +# Pre-create root's ansible remote_tmp dir before handing off to the real container command +# (systemd via /usr/sbin/init). This runs synchronously before the container is reported as +# started, so it can never race the first ansible connection's own mkdir of this same path. +install -d -m 0700 /root/.ansible/tmp + +exec "$@" diff --git a/extensions/molecule/rollback/molecule.yml b/extensions/molecule/rollback/molecule.yml new file mode 100644 index 0000000..37d5320 --- /dev/null +++ b/extensions/molecule/rollback/molecule.yml @@ -0,0 +1,73 @@ +--- +driver: + name: docker +platforms: + - name: ssh-key-rotation-rollback-ubuntu2204 + image: geerlingguy/docker-ubuntu2204-ansible:latest + command: /usr/sbin/init + privileged: true + cgroupns_mode: host + pre_build_image: false + tmpfs: + - /run + - /run/lock + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:rw + groups: + - rotate + - name: ssh-key-rotation-rollback-rockylinux9 + image: geerlingguy/docker-rockylinux9-ansible:latest + command: /usr/sbin/init + privileged: true + cgroupns_mode: host + pre_build_image: false + tmpfs: + - /run + - /run/lock + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:rw + groups: + - rotate + - name: ssh-key-rotation-rollback-rockylinux10 + image: geerlingguy/docker-rockylinux10-ansible:latest + command: /usr/sbin/init + privileged: true + cgroupns_mode: host + pre_build_image: false + tmpfs: + - /run + - /run/lock + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:rw + groups: + - rotate +provisioner: + name: ansible + env: + ANSIBLE_ROLES_PATH: "../../../roles" + inventory: + group_vars: + # The docker connection plugin doesn't invoke a login shell, so `~` in the default + # remote_tmp path doesn't reliably expand for the "rotate" containers. Pin it to the + # absolute path molecule-entrypoint.sh already pre-creates for root before systemd + # starts. Scoped to this group only - localhost (used by molecule's own internal + # playbooks and the key-generation play) must keep its normal, non-root remote_tmp. + rotate: + ansible_remote_tmp: /root/.ansible/tmp + # These containers are entered as root. Saying so explicitly means Ansible skips + # become for the role's `become: true` tasks (become_user root == remote_user root) + # instead of shelling out to sudo. Without this, sudo runs and fails on images whose + # pam/shadow setup can't serve an account lookup inside a container (seen on + # rockylinux9: "PAM account management error: Authentication service cannot retrieve + # authentication info"), which has nothing to do with what these tests are proving. + ansible_user: root +scenario: + test_sequence: + - syntax + - create + - prepare + - converge + - verify + - destroy +verifier: + name: ansible diff --git a/extensions/molecule/rollback/prepare.yml b/extensions/molecule/rollback/prepare.yml new file mode 100644 index 0000000..1968423 --- /dev/null +++ b/extensions/molecule/rollback/prepare.yml @@ -0,0 +1,90 @@ +--- +# Same environment setup as molecule/default: ephemeral keys + a pre-authorized "old" key + +# a generated inventory. This scenario needs THREE keypairs: seed_key (pre-existing access), +# key_a (installed by a first, successful rotation used as the "known-good" checkpoint), and +# key_b (the second rotation, deliberately broken partway through verify, to prove the +# rescue block restores access to key_a). + +- name: Prepare | Generate ephemeral keys and the test inventory on the control node + hosts: localhost + connection: local + gather_facts: false + vars: + ssh_key_rotation_test_dir: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}" + tasks: + - name: Prepare | Generate the ephemeral SEED (pre-existing) keypair + ansible.builtin.command: "ssh-keygen -t rsa -b 3072 -N '' -f {{ ssh_key_rotation_test_dir }}/seed_key" + args: + creates: "{{ ssh_key_rotation_test_dir }}/seed_key" + + - name: Prepare | Generate the ephemeral KEY_A keypair (first, successful rotation) + ansible.builtin.command: "ssh-keygen -t ed25519 -N '' -f {{ ssh_key_rotation_test_dir }}/key_a" + args: + creates: "{{ ssh_key_rotation_test_dir }}/key_a" + + - name: Prepare | Generate the ephemeral KEY_B keypair (second, deliberately broken rotation) + ansible.builtin.command: "ssh-keygen -t ed25519 -N '' -f {{ ssh_key_rotation_test_dir }}/key_b" + args: + creates: "{{ ssh_key_rotation_test_dir }}/key_b" + +- name: Prepare | Seed the SEED key into each container's authorized_keys + hosts: rotate + gather_facts: true + vars: + ssh_key_rotation_test_dir: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}" + # openssh-server is installed at Dockerfile build time (see Dockerfile.j2), not here. The + # service/unit name still differs by family: Debian/Ubuntu: ssh, RedHat/Rocky: sshd. + ssh_key_rotation_sshd_service: "{{ 'sshd' if ansible_facts['os_family'] == 'RedHat' else 'ssh' }}" + tasks: + - name: Prepare | Ensure sshd is running + ansible.builtin.systemd: + name: "{{ ssh_key_rotation_sshd_service }}" + state: started + enabled: true + + - name: Prepare | Ensure PermitRootLogin allows key auth + ansible.builtin.lineinfile: + path: /etc/ssh/sshd_config + regexp: '^#?\s*PermitRootLogin' + line: 'PermitRootLogin prohibit-password' + notify: Restart sshd for prepare + + - name: Prepare | Install the ephemeral SEED public key as pre-existing authorized access + ansible.posix.authorized_key: + user: root + key: "{{ lookup('file', ssh_key_rotation_test_dir + '/seed_key.pub') }}" + state: present + + - name: Prepare | Discover this container's IP address on the docker network + ansible.builtin.command: hostname -I + changed_when: false + register: ssh_key_rotation_container_ip + + - name: Prepare | Record this container's IP for the generated inventory + ansible.builtin.set_fact: + ssh_key_rotation_container_ip: "{{ ssh_key_rotation_container_ip.stdout.split() | first }}" + + - name: Prepare | Ensure the [rotate] group header is present in the generated inventory + ansible.builtin.lineinfile: + path: "{{ ssh_key_rotation_test_dir }}/rotation_inventory.ini" + line: "[rotate]" + insertbefore: BOF + create: true + mode: "0600" + delegate_to: localhost + run_once: true + + - name: Prepare | Append this host to the generated rotation inventory + ansible.builtin.lineinfile: + path: "{{ ssh_key_rotation_test_dir }}/rotation_inventory.ini" + line: >- + {{ inventory_hostname }} ansible_host={{ ssh_key_rotation_container_ip }} ansible_user=root + ansible_ssh_common_args='-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null' + insertafter: EOF + delegate_to: localhost + + handlers: + - name: Restart sshd for prepare + ansible.builtin.systemd: + name: "{{ ssh_key_rotation_sshd_service }}" + state: restarted diff --git a/extensions/molecule/rollback/verify.yml b/extensions/molecule/rollback/verify.yml new file mode 100644 index 0000000..9b7d993 --- /dev/null +++ b/extensions/molecule/rollback/verify.yml @@ -0,0 +1,87 @@ +--- +# Confirms the rescue: block in roles/ssh_key_rotation/tasks/verify.yml actually restored +# access after rotation 2 was deliberately broken: key_a (the pre-failure state) must still +# work, authorized_keys must match the pre-failure checkpoint, and sshd must still be running. + +- name: Verify | Confirm key_a still authenticates after the failed/rolled-back rotation + hosts: rotate + gather_facts: false + vars: + ssh_key_rotation_test_dir: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}" + ssh_key_rotation_verify_ssh_opts: >- + -o BatchMode=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null + -o IdentitiesOnly=yes -o ConnectTimeout=15 + tasks: + - name: Verify | Discover this container's IP address for the real SSH check + ansible.builtin.command: hostname -I + changed_when: false + register: ssh_key_rotation_verify_container_ip + + # Has to be a real ssh client: molecule's docker driver reaches this group over + # ansible_connection=community.docker.docker, which ignores ansible_ssh_private_key_file, so an + # ansible.builtin.ping here succeeds no matter which key is passed (it returns "pong" even for a + # nonexistent key file) and would prove nothing about key_a still working. This is THE assertion + # the whole rollback scenario exists to make, so it must exercise the real transport. + - name: Verify | key_a authenticates over real SSH after the rollback + ansible.builtin.command: >- + ssh -i {{ ssh_key_rotation_test_dir }}/key_a {{ ssh_key_rotation_verify_ssh_opts }} + root@{{ ssh_key_rotation_verify_container_ip.stdout.split() | first }} true + delegate_to: localhost + changed_when: false + + - name: Verify | sshd is running + ansible.builtin.command: systemctl is-active sshd + become: true + changed_when: false + + - name: Verify | Read authorized_keys after the failed rotation + ansible.builtin.slurp: + src: /root/.ssh/authorized_keys + register: ssh_key_rotation_post_failure_authorized_keys + + - name: Verify | Load the pre-failure checkpoint + ansible.builtin.slurp: + src: "{{ ssh_key_rotation_test_dir }}/checkpoint_authorized_keys.{{ inventory_hostname }}" + delegate_to: localhost + register: ssh_key_rotation_checkpoint_authorized_keys + + # Superset, not exact equality. The checkpoint is taken after rotation 1, i.e. BEFORE rotation + # 2's install stage adds key_b, whereas the rescue block restores the backup taken at the start + # of the VERIFY stage - which already contains key_b. Demanding an exact match would therefore + # require the rescue to undo the install stage, which it does not claim to do and should not: + # install is purely additive (it adds the new key and removes nothing), so it leaves nothing + # unsafe behind. The property that actually matters is that no key authorized before the failed + # rotation was lost - i.e. the known-good access is back. key_b remaining is expected and benign: + # it is a key the operator deliberately installed and holds the private half of. + - name: Verify | No authorized key from the pre-failure checkpoint was lost in the rollback + vars: + ssh_key_rotation_checkpoint_key_lines: >- + {{ (ssh_key_rotation_checkpoint_authorized_keys.content | b64decode).splitlines() + | map('trim') | reject('eq', '') | list }} + ssh_key_rotation_post_failure_key_lines: >- + {{ (ssh_key_rotation_post_failure_authorized_keys.content | b64decode).splitlines() + | map('trim') | reject('eq', '') | list }} + ansible.builtin.assert: + that: >- + ssh_key_rotation_checkpoint_key_lines + | difference(ssh_key_rotation_post_failure_key_lines) | length == 0 + fail_msg: >- + authorized_keys is missing key(s) that were present before the failed rotation - the + rescue block did not restore access. Missing: + {{ ssh_key_rotation_checkpoint_key_lines + | difference(ssh_key_rotation_post_failure_key_lines) }} + success_msg: >- + Every key authorized before the failed rotation is present again after the rollback + ({{ ssh_key_rotation_post_failure_key_lines | length }} key(s) now authorized). + + # Literal `in`, never `is search`: `search` treats its argument as a REGEX and a base64 key blob + # routinely contains `+` (a quantifier), so the pattern fails to match a key that IS present. + - name: Verify | key_a specifically is back in authorized_keys + ansible.builtin.assert: + that: >- + lookup('file', ssh_key_rotation_test_dir + '/key_a.pub').split()[1] + in (ssh_key_rotation_post_failure_authorized_keys.content | b64decode) + fail_msg: >- + key_a (the known-good key from the successful first rotation) is NOT in authorized_keys + after the rollback - the rescue block failed to restore the pre-failure access. + success_msg: "key_a is back in authorized_keys after the rollback." diff --git a/galaxy.yml b/galaxy.yml index 1725230..0ca32e7 100644 --- a/galaxy.yml +++ b/galaxy.yml @@ -18,3 +18,12 @@ tags: - infrastructure dependencies: ansible.posix: ">=1.3.0" +build_ignore: + - molecule + - test_vars.yml + - test_rsa + - test_rsa.pub + - test_ecdsa + - test_ecdsa.pub + - test_ed25519 + - test_ed25519.pub diff --git a/roles/ssh_key_rotation/tasks/manage_crypto_policy.yml b/roles/ssh_key_rotation/tasks/manage_crypto_policy.yml index 6e2dc1b..a497703 100644 --- a/roles/ssh_key_rotation/tasks/manage_crypto_policy.yml +++ b/roles/ssh_key_rotation/tasks/manage_crypto_policy.yml @@ -12,7 +12,7 @@ # ssh_key_rotation_crypto_policy_notify_reload - notify the "Reload sshd" handler once applied (default true; # validate.yml sets this false, there's no target sshd to reload on the control node) - name: "Manage crypto policy | Check whether the crypto-policies tool is present on {{ ssh_key_rotation_crypto_policy_context }}" - when: ssh_key_rotation_manage_crypto_policy + when: ssh_key_rotation_manage_crypto_policy | bool ansible.builtin.command: command -v update-crypto-policies changed_when: false failed_when: false @@ -20,7 +20,7 @@ - name: "Manage crypto policy | Warn if manage_crypto_policy is set but the crypto-policies tool is missing on {{ ssh_key_rotation_crypto_policy_context }}" when: - - ssh_key_rotation_manage_crypto_policy + - ssh_key_rotation_manage_crypto_policy | bool - ssh_key_rotation_crypto_policies_tool.rc != 0 ansible.builtin.debug: msg: >- @@ -30,7 +30,7 @@ - name: "Manage crypto policy | Check the current system-wide crypto policy on {{ ssh_key_rotation_crypto_policy_context }}" when: - - ssh_key_rotation_manage_crypto_policy + - ssh_key_rotation_manage_crypto_policy | bool - ssh_key_rotation_crypto_policies_tool.rc == 0 ansible.builtin.command: update-crypto-policies --show changed_when: false @@ -39,7 +39,7 @@ # Merge the desired crypto policy onto this host's CURRENT policy, rather than replacing it outright. - name: "Manage crypto policy | Merge the desired crypto policy onto the CURRENT policy on {{ ssh_key_rotation_crypto_policy_context }}" when: - - ssh_key_rotation_manage_crypto_policy + - ssh_key_rotation_manage_crypto_policy | bool - ssh_key_rotation_crypto_policies_tool.rc == 0 - ssh_key_rotation_crypto_policy_add_modules | length > 0 ansible.builtin.set_fact: @@ -51,7 +51,7 @@ - name: "Manage crypto policy | Fall back to crypto_policy_setting as the desired crypto policy for {{ ssh_key_rotation_crypto_policy_context }}" when: - - ssh_key_rotation_manage_crypto_policy + - ssh_key_rotation_manage_crypto_policy | bool - ssh_key_rotation_crypto_policies_tool.rc == 0 - ssh_key_rotation_crypto_policy_add_modules | length == 0 ansible.builtin.set_fact: @@ -60,7 +60,7 @@ # BASE:MODULE (e.g. FIPS:PQ) needs a MODULE.pmod under /usr/share or /etc crypto-policies modules dirs; see README's crypto-policy section for why. - name: "Manage crypto policy | Discover available crypto-policy subpolicy modules on {{ ssh_key_rotation_crypto_policy_context }}" when: - - ssh_key_rotation_manage_crypto_policy + - ssh_key_rotation_manage_crypto_policy | bool - ssh_key_rotation_crypto_policies_tool.rc == 0 - "':' in ssh_key_rotation_crypto_policy_desired" ansible.builtin.find: @@ -73,7 +73,7 @@ - name: "Manage crypto policy | Determine which subpolicy modules the desired crypto policy is missing on {{ ssh_key_rotation_crypto_policy_context }}" when: - - ssh_key_rotation_manage_crypto_policy + - ssh_key_rotation_manage_crypto_policy | bool - ssh_key_rotation_crypto_policies_tool.rc == 0 - "':' in ssh_key_rotation_crypto_policy_desired" ansible.builtin.set_fact: @@ -95,7 +95,7 @@ - name: "Manage crypto policy | Apply the desired crypto policy on {{ ssh_key_rotation_crypto_policy_context }}" when: - - ssh_key_rotation_manage_crypto_policy + - ssh_key_rotation_manage_crypto_policy | bool - ssh_key_rotation_crypto_policies_tool.rc == 0 - ssh_key_rotation_crypto_policy_current.stdout != ssh_key_rotation_crypto_policy_desired ansible.builtin.command: "update-crypto-policies --set {{ ssh_key_rotation_crypto_policy_desired }}" diff --git a/roles/ssh_key_rotation/tasks/verify.yml b/roles/ssh_key_rotation/tasks/verify.yml index 458e159..17afce2 100644 --- a/roles/ssh_key_rotation/tasks/verify.yml +++ b/roles/ssh_key_rotation/tasks/verify.yml @@ -37,7 +37,7 @@ state: absent - name: Verify | (Optional) Make authorized_keys exclusive to the PwC key - when: ssh_key_rotation_make_exclusive + when: ssh_key_rotation_make_exclusive | bool ansible.posix.authorized_key: user: "{{ ssh_key_rotation_target_user }}" key: "{{ lookup('file', new_public_key_file) }}" @@ -47,7 +47,7 @@ # Explicit, deterministically-named backup (same pattern as authorized_keys above), so rescue: below # knows exactly which file to restore, rather than chasing lineinfile's own auto-named backup files. - name: Verify | Back up sshd_config before disabling legacy auth - when: ssh_key_rotation_disable_password_auth or ssh_key_rotation_disable_kbd_interactive + when: (ssh_key_rotation_disable_password_auth | bool) or (ssh_key_rotation_disable_kbd_interactive | bool) ansible.builtin.copy: src: /etc/ssh/sshd_config dest: "/etc/ssh/sshd_config.bak-{{ ansible_date_time.iso8601_basic_short }}" @@ -57,7 +57,7 @@ register: ssh_key_rotation_sshd_config_backup - name: Verify | Disable password authentication (the old way of connecting) - when: ssh_key_rotation_disable_password_auth + when: ssh_key_rotation_disable_password_auth | bool ansible.builtin.lineinfile: path: /etc/ssh/sshd_config regexp: '^#?\s*PasswordAuthentication' @@ -68,7 +68,7 @@ notify: Reload sshd - name: Verify | Disable keyboard-interactive authentication - when: ssh_key_rotation_disable_kbd_interactive + when: ssh_key_rotation_disable_kbd_interactive | bool ansible.builtin.lineinfile: path: /etc/ssh/sshd_config regexp: '^#?\s*KbdInteractiveAuthentication' @@ -104,13 +104,23 @@ mode: preserve become: true + # A failed reload must not abort the rollback. authorized_keys and sshd_config are already + # restored on disk above, and sshd reads authorized_keys per-connection, so access is back even + # if the daemon never re-read its config. rescue: has no rescue of its own, so letting this fail + # would skip the two tasks that matter most here: the connectivity check below (the real access + # gate) and the diagnostic that tells the operator what was restored. Reported in that message + # instead. Note the reload can fail for the very same reason the block did - both use + # ssh_key_rotation_sshd_service - so this is the expected path, not an edge case. - name: Rollback | Reload sshd with the restored configuration when: ssh_key_rotation_sshd_config_backup is defined and ssh_key_rotation_sshd_config_backup is succeeded ansible.builtin.service: name: "{{ ssh_key_rotation_sshd_service }}" state: reloaded become: true + register: ssh_key_rotation_rollback_reload + ignore_errors: true + # Stays fatal on purpose: this is what proves access survived the rollback. - name: Rollback | Confirm connectivity still works after rollback ansible.builtin.ping: @@ -125,3 +135,10 @@ if (ssh_key_rotation_sshd_config_backup is defined and ssh_key_rotation_sshd_config_backup is succeeded) else '(sshd_config was not touched)' }}. The old key and prior sshd_config are back in place; nothing was left half-changed. + {{ ('WARNING: sshd could not be reloaded during the rollback (' + ~ (ssh_key_rotation_rollback_reload.msg | default('unknown error')) + ~ ') - the restored sshd_config is on disk but is NOT yet active. Reload or restart' + ~ ' sshd manually to activate it. Key-based access is unaffected, since sshd reads' + ~ ' authorized_keys per connection.') + if (ssh_key_rotation_rollback_reload is defined and ssh_key_rotation_rollback_reload is failed) + else '' }} From 0803c72a822492e7a897a873b536dc403cd4b92d Mon Sep 17 00:00:00 2001 From: Mark Bolwell Date: Wed, 29 Jul 2026 18:18:16 +0100 Subject: [PATCH 02/18] Added yaml header --- .github/workflows/ci.yml | 1 + .github/workflows/molecule.yml | 1 + extensions/molecule/default/converge.yml | 1 + extensions/molecule/default/molecule.yml | 1 + extensions/molecule/default/prepare.yml | 1 + extensions/molecule/default/verify.yml | 1 + extensions/molecule/rollback/converge.yml | 1 + extensions/molecule/rollback/molecule.yml | 1 + extensions/molecule/rollback/prepare.yml | 1 + extensions/molecule/rollback/verify.yml | 1 + galaxy.yml | 1 + meta/runtime.yml | 1 + requirements.yml | 1 + 13 files changed, 13 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5ff0f42..8ba435c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,4 +1,5 @@ --- + name: CI on: diff --git a/.github/workflows/molecule.yml b/.github/workflows/molecule.yml index 367511d..d872d40 100644 --- a/.github/workflows/molecule.yml +++ b/.github/workflows/molecule.yml @@ -1,4 +1,5 @@ --- + name: Molecule on: diff --git a/extensions/molecule/default/converge.yml b/extensions/molecule/default/converge.yml index 2faa06d..f2661da 100644 --- a/extensions/molecule/default/converge.yml +++ b/extensions/molecule/default/converge.yml @@ -1,4 +1,5 @@ --- + # Runs the collection's real entry point (playbooks/rotate.yml, all 3 stages) against the # generated inventory/vars from prepare.yml -- this is what proves the shipped playbook works # end to end, not just the role in isolation. Then runs it again, unchanged, to prove diff --git a/extensions/molecule/default/molecule.yml b/extensions/molecule/default/molecule.yml index 11793f4..82fbe10 100644 --- a/extensions/molecule/default/molecule.yml +++ b/extensions/molecule/default/molecule.yml @@ -1,4 +1,5 @@ --- + driver: name: docker platforms: diff --git a/extensions/molecule/default/prepare.yml b/extensions/molecule/default/prepare.yml index a6ed81d..2e6f859 100644 --- a/extensions/molecule/default/prepare.yml +++ b/extensions/molecule/default/prepare.yml @@ -1,4 +1,5 @@ --- + # Prepares each Molecule-managed container as a realistic rotation target: an ephemeral # "old" keypair pre-authorized (simulating existing access before install.yml ever runs), # an ephemeral "new" keypair generated but NOT yet authorized, and a generated inventory + diff --git a/extensions/molecule/default/verify.yml b/extensions/molecule/default/verify.yml index 7e8c5ba..0d95b6e 100644 --- a/extensions/molecule/default/verify.yml +++ b/extensions/molecule/default/verify.yml @@ -1,4 +1,5 @@ --- + # Confirms the rotation actually did what it claims: the new key authenticates, the old key # is rejected, authorized_keys/sshd_config reflect the role's (default) settings. diff --git a/extensions/molecule/rollback/converge.yml b/extensions/molecule/rollback/converge.yml index 1a74fe6..e9d5533 100644 --- a/extensions/molecule/rollback/converge.yml +++ b/extensions/molecule/rollback/converge.yml @@ -1,4 +1,5 @@ --- + # Runs TWO rotations against the container(s): # 1. seed_key -> key_a: a normal, fully successful rotation (this scenario leaves password # and keyboard-interactive auth ENABLED on purpose, so run 2 below has something new to diff --git a/extensions/molecule/rollback/molecule.yml b/extensions/molecule/rollback/molecule.yml index 37d5320..0b140b3 100644 --- a/extensions/molecule/rollback/molecule.yml +++ b/extensions/molecule/rollback/molecule.yml @@ -1,4 +1,5 @@ --- + driver: name: docker platforms: diff --git a/extensions/molecule/rollback/prepare.yml b/extensions/molecule/rollback/prepare.yml index 1968423..96df43c 100644 --- a/extensions/molecule/rollback/prepare.yml +++ b/extensions/molecule/rollback/prepare.yml @@ -1,4 +1,5 @@ --- + # Same environment setup as molecule/default: ephemeral keys + a pre-authorized "old" key + # a generated inventory. This scenario needs THREE keypairs: seed_key (pre-existing access), # key_a (installed by a first, successful rotation used as the "known-good" checkpoint), and diff --git a/extensions/molecule/rollback/verify.yml b/extensions/molecule/rollback/verify.yml index 9b7d993..1e05c7b 100644 --- a/extensions/molecule/rollback/verify.yml +++ b/extensions/molecule/rollback/verify.yml @@ -1,4 +1,5 @@ --- + # Confirms the rescue: block in roles/ssh_key_rotation/tasks/verify.yml actually restored # access after rotation 2 was deliberately broken: key_a (the pre-failure state) must still # work, authorized_keys must match the pre-failure checkpoint, and sshd must still be running. diff --git a/galaxy.yml b/galaxy.yml index 0ca32e7..9063b43 100644 --- a/galaxy.yml +++ b/galaxy.yml @@ -1,4 +1,5 @@ --- + namespace: krameff name: ssh_key_rotation version: 1.0.0 diff --git a/meta/runtime.yml b/meta/runtime.yml index 114725c..23d890c 100644 --- a/meta/runtime.yml +++ b/meta/runtime.yml @@ -1,2 +1,3 @@ --- + requires_ansible: ">=2.18.0" diff --git a/requirements.yml b/requirements.yml index 18b01ef..f36c7bb 100644 --- a/requirements.yml +++ b/requirements.yml @@ -1,4 +1,5 @@ --- + # Ansible Galaxy requirements for the SSH Key Rotation collection # Install with: ansible-galaxy install -r requirements.yml From 167b488d4cfcc4ab59b8819f881c789829e088d3 Mon Sep 17 00:00:00 2001 From: Mark Bolwell Date: Fri, 31 Jul 2026 01:54:12 +0100 Subject: [PATCH 03/18] Updates to molecule layout --- .github/workflows/molecule.yml | 11 +- .gitignore | 1 + CHANGELOG.md | 8 ++ .../molecule/default/molecule-entrypoint.sh | 9 -- extensions/molecule/default/molecule.yml | 59 +++++---- extensions/molecule/default/prepare.yml | 112 +++-------------- extensions/molecule/default/verify.yml | 11 +- .../{default => resources}/Dockerfile.j2 | 17 +-- extensions/molecule/resources/prepare.yml | 119 ++++++++++++++++++ extensions/molecule/rollback/Dockerfile.j2 | 34 ----- .../molecule/rollback/molecule-entrypoint.sh | 9 -- extensions/molecule/rollback/molecule.yml | 58 +++++---- extensions/molecule/rollback/prepare.yml | 106 +++------------- extensions/molecule/rollback/verify.yml | 8 +- 14 files changed, 251 insertions(+), 311 deletions(-) delete mode 100755 extensions/molecule/default/molecule-entrypoint.sh rename extensions/molecule/{default => resources}/Dockerfile.j2 (63%) create mode 100644 extensions/molecule/resources/prepare.yml delete mode 100644 extensions/molecule/rollback/Dockerfile.j2 delete mode 100755 extensions/molecule/rollback/molecule-entrypoint.sh diff --git a/.github/workflows/molecule.yml b/.github/workflows/molecule.yml index d872d40..8eb6e93 100644 --- a/.github/workflows/molecule.yml +++ b/.github/workflows/molecule.yml @@ -7,6 +7,11 @@ on: branches: [main, devel] pull_request: branches: [main, devel] + # The test platforms track the geerlingguy images by :latest, so the base images move + # underneath this suite between commits. A weekly run means that drift surfaces as its own + # red build rather than as a mystery failure on whatever unrelated PR is open at the time. + schedule: + - cron: "0 6 * * 1" workflow_dispatch: jobs: @@ -30,9 +35,9 @@ jobs: python -m pip install --upgrade pip pip install "ansible>=2.18" "molecule>=25.0" "molecule-plugins[docker]" - - name: Install collection dependencies - run: ansible-galaxy install -r requirements.yml - + # No separate ansible-galaxy step: the scenarios run `dependency` as the first step of + # their test sequence, so collection dependencies get installed the same way here and on + # a developer's machine. - name: Run Molecule scenario run: molecule test -s ${{ matrix.scenario }} env: diff --git a/.gitignore b/.gitignore index 9b78f75..f0b696f 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,7 @@ venv/ .ansible/ ansible_collections/ inventory.ini +.mcp* # SSH key material - QUICKSTART.md has users generate keys in this directory; # never commit private keys. Public keys (*.pub) are safe and stay tracked. diff --git a/CHANGELOG.md b/CHANGELOG.md index 15b366c..7c29dbe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,11 +10,19 @@ - Molecule key-authentication checks ran over the container connection, which ignores SSH keys and so could never detect a failure; they now drive a real ssh client. - Molecule idempotency check required `changed=0` from the timestamped backup tasks, which cannot be idempotent by design; it now allows those and fails on anything else. - Molecule rollback check compared `authorized_keys` against a checkpoint taken before the install stage, demanding the rescue undo work it never performs. +- Molecule rollback "sshd is running" check hardcoded the `sshd` unit, which resolves on Ubuntu only through an alias; it now uses the same family lookup as prepare. +- Molecule set `ANSIBLE_ROLES_PATH` to a scenario-relative path, but Molecule runs with the project directory as its cwd, so it resolved outside the repo and overrode the `roles_path` in `ansible.cfg` that makes a bare `ssh_key_rotation` role name resolve. Removed, so `ansible.cfg` applies. +- The default scenario's first verify play was named as a new-key authentication check but runs over the container connection, which ignores SSH keys, so it could never fail; it is now named and documented as the reachability check it actually is. The key-authentication proof was already in the following play, which drives a real ssh client. ### Changed - Molecule scenarios moved from `molecule/` to `extensions/molecule/`, which also corrects the relative roles path. - Molecule test images install `openssh-server` at build time rather than during prepare. +- Both Molecule scenarios now share one `extensions/molecule/resources/Dockerfile.j2` and one `resources/prepare.yml`; each scenario's `prepare.yml` is a short wrapper that supplies only its own keypairs and which one to pre-authorize. +- Root's Ansible `remote_tmp` directory is created during the image build instead of by an entrypoint wrapper, so it exists before the container ever starts and `molecule-entrypoint.sh` is gone from both scenarios. +- Molecule platform definitions use a YAML anchor rather than repeating three near-identical blocks per scenario. +- Molecule test sequences start with `dependency` and `destroy`: collection dependencies are installed the same way locally and in CI (the workflow's separate `ansible-galaxy` step is gone), and a container left behind by an aborted run can no longer be reused with keys already in its `authorized_keys`. +- The Molecule workflow also runs weekly, so drift in the `:latest` test images surfaces on its own build rather than on an unrelated pull request. ## [Initial] diff --git a/extensions/molecule/default/molecule-entrypoint.sh b/extensions/molecule/default/molecule-entrypoint.sh deleted file mode 100755 index d0a681e..0000000 --- a/extensions/molecule/default/molecule-entrypoint.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash -set -euo pipefail - -# Pre-create root's ansible remote_tmp dir before handing off to the real container command -# (systemd via /usr/sbin/init). This runs synchronously before the container is reported as -# started, so it can never race the first ansible connection's own mkdir of this same path. -install -d -m 0700 /root/.ansible/tmp - -exec "$@" diff --git a/extensions/molecule/default/molecule.yml b/extensions/molecule/default/molecule.yml index 82fbe10..899f74c 100644 --- a/extensions/molecule/default/molecule.yml +++ b/extensions/molecule/default/molecule.yml @@ -3,8 +3,13 @@ driver: name: docker platforms: - - name: ssh-key-rotation-ubuntu2204 + # The first platform carries the settings every platform shares; the rest merge it and + # override only name/image. The Dockerfile itself is shared across scenarios too, hence the + # relative path (the docker driver resolves `dockerfile` against this scenario directory). + - &platform_defaults + name: ssh-key-rotation-ubuntu2204 image: geerlingguy/docker-ubuntu2204-ansible:latest + dockerfile: ../resources/Dockerfile.j2 command: /usr/sbin/init privileged: true cgroupns_mode: host @@ -16,43 +21,32 @@ platforms: - /sys/fs/cgroup:/sys/fs/cgroup:rw groups: - rotate - - name: ssh-key-rotation-rockylinux9 + - <<: *platform_defaults + name: ssh-key-rotation-rockylinux9 image: geerlingguy/docker-rockylinux9-ansible:latest - command: /usr/sbin/init - privileged: true - cgroupns_mode: host - pre_build_image: false - tmpfs: - - /run - - /run/lock - volumes: - - /sys/fs/cgroup:/sys/fs/cgroup:rw - groups: - - rotate - - name: ssh-key-rotation-rockylinux10 + - <<: *platform_defaults + name: ssh-key-rotation-rockylinux10 image: geerlingguy/docker-rockylinux10-ansible:latest - command: /usr/sbin/init - privileged: true - cgroupns_mode: host - pre_build_image: false - tmpfs: - - /run - - /run/lock - volumes: - - /sys/fs/cgroup:/sys/fs/cgroup:rw - groups: - - rotate +dependency: + # Mirrors exactly what CI used to run as a separate step, so a local `molecule test` and a + # CI run install the same collection dependencies. Molecule chdirs to the project directory + # (the collection root) before running a sequence, so this relative path is correct. + name: shell + command: ansible-galaxy install -r requirements.yml provisioner: name: ansible - env: - ANSIBLE_ROLES_PATH: "../../../roles" + # Deliberately no ANSIBLE_ROLES_PATH here. Molecule runs with the project directory as its + # cwd, not the scenario directory, so a scenario-relative value resolves outside the repo and, + # worse, overrides the roles_path in the collection's own ansible.cfg - which is what makes a + # bare `ssh_key_rotation` role name resolve. Leaving it unset lets ansible.cfg do its job. + # The converge that shells out to playbooks/rotate.yml sets its own absolute path. inventory: group_vars: # The docker connection plugin doesn't invoke a login shell, so `~` in the default # remote_tmp path doesn't reliably expand for the "rotate" containers. Pin it to the - # absolute path molecule-entrypoint.sh already pre-creates for root before systemd - # starts. Scoped to this group only - localhost (used by molecule's own internal - # playbooks and the key-generation play) must keep its normal, non-root remote_tmp. + # absolute path resources/Dockerfile.j2 already creates for root in the image. Scoped to + # this group only - localhost (used by molecule's own internal playbooks and the + # key-generation play) must keep its normal, non-root remote_tmp. rotate: ansible_remote_tmp: /root/.ansible/tmp # These containers are entered as root. Saying so explicitly means Ansible skips @@ -64,6 +58,11 @@ provisioner: ansible_user: root scenario: test_sequence: + - dependency + # Leading destroy: without it a container left behind by an aborted run is reused, and its + # authorized_keys already holds the rotated-in key. That silently breaks this suite's core + # premise (only the seeded old key is authorized before converge runs). + - destroy - syntax - create - prepare diff --git a/extensions/molecule/default/prepare.yml b/extensions/molecule/default/prepare.yml index 2e6f859..9b5f695 100644 --- a/extensions/molecule/default/prepare.yml +++ b/extensions/molecule/default/prepare.yml @@ -1,100 +1,20 @@ --- -# Prepares each Molecule-managed container as a realistic rotation target: an ephemeral -# "old" keypair pre-authorized (simulating existing access before install.yml ever runs), -# an ephemeral "new" keypair generated but NOT yet authorized, and a generated inventory + -# vars file that converge.yml uses to invoke the real playbooks/rotate.yml over actual SSH -# (not the Molecule docker connection plugin, since that's not what the role uses in production). +# This scenario rotates one key to another, so it needs two keypairs: old_key (pre-authorized, +# simulating existing access) and new_key (generated but NOT yet authorized). It drives +# playbooks/rotate.yml as a single ansible-playbook invocation, so it also needs the generated +# rotation_vars.yml extra-vars file. Everything else is in resources/prepare.yml. -- name: Prepare | Generate ephemeral keys and the test inventory on the control node - hosts: localhost - connection: local - gather_facts: false +- name: Prepare | Set up the containers as rotation targets + ansible.builtin.import_playbook: ../resources/prepare.yml vars: - ssh_key_rotation_test_dir: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}" - tasks: - - name: Prepare | Generate the ephemeral OLD (RSA) keypair - ansible.builtin.command: "ssh-keygen -t rsa -b 3072 -N '' -f {{ ssh_key_rotation_test_dir }}/old_key" - args: - creates: "{{ ssh_key_rotation_test_dir }}/old_key" - - - name: Prepare | Generate the ephemeral NEW (ED25519) keypair - ansible.builtin.command: "ssh-keygen -t ed25519 -N '' -f {{ ssh_key_rotation_test_dir }}/new_key" - args: - creates: "{{ ssh_key_rotation_test_dir }}/new_key" - - - name: Prepare | Write the generated rotation vars file used by converge.yml - ansible.builtin.copy: - dest: "{{ ssh_key_rotation_test_dir }}/rotation_vars.yml" - mode: "0600" - content: | - --- - old_private_key: "{{ ssh_key_rotation_test_dir }}/old_key" - old_public_key_file: "{{ ssh_key_rotation_test_dir }}/old_key.pub" - new_private_key: "{{ ssh_key_rotation_test_dir }}/new_key" - new_public_key_file: "{{ ssh_key_rotation_test_dir }}/new_key.pub" - ssh_key_rotation_target_user: root - -- name: Prepare | Seed the OLD key into each container's authorized_keys (simulates pre-existing access) - hosts: rotate - gather_facts: true - vars: - ssh_key_rotation_test_dir: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}" - # openssh-server is installed at Dockerfile build time (see Dockerfile.j2), not here. The - # service/unit name still differs by family: Debian/Ubuntu: ssh, RedHat/Rocky: sshd. - ssh_key_rotation_sshd_service: "{{ 'sshd' if ansible_facts['os_family'] == 'RedHat' else 'ssh' }}" - tasks: - - name: Prepare | Ensure sshd is running - ansible.builtin.systemd: - name: "{{ ssh_key_rotation_sshd_service }}" - state: started - enabled: true - - - name: Prepare | Ensure PermitRootLogin allows key auth (test images default to prohibit-password already) - ansible.builtin.lineinfile: - path: /etc/ssh/sshd_config - regexp: '^#?\s*PermitRootLogin' - line: 'PermitRootLogin prohibit-password' - notify: Restart sshd for prepare - - - name: Prepare | Install the ephemeral OLD public key as pre-existing authorized access - ansible.posix.authorized_key: - user: root - key: "{{ lookup('file', ssh_key_rotation_test_dir + '/old_key.pub') }}" - state: present - - - name: Prepare | Discover this container's IP address on the docker network - ansible.builtin.command: hostname -I - changed_when: false - register: ssh_key_rotation_container_ip - - - name: Prepare | Record this container's IP for the generated inventory - ansible.builtin.set_fact: - ssh_key_rotation_container_ip: "{{ ssh_key_rotation_container_ip.stdout.split() | first }}" - - - name: Prepare | Append this host to the generated rotation inventory - ansible.builtin.lineinfile: - path: "{{ ssh_key_rotation_test_dir }}/rotation_inventory.ini" - create: true - mode: "0600" - line: >- - {{ inventory_hostname }} ansible_host={{ ssh_key_rotation_container_ip }} ansible_user=root - ansible_ssh_common_args='-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null' - insertafter: EOF - delegate_to: localhost - - - name: Prepare | Ensure the [rotate] group header is present in the generated inventory - ansible.builtin.lineinfile: - path: "{{ ssh_key_rotation_test_dir }}/rotation_inventory.ini" - line: "[rotate]" - insertbefore: BOF - create: true - mode: "0600" - delegate_to: localhost - run_once: true - - handlers: - - name: Restart sshd for prepare - ansible.builtin.systemd: - name: "{{ ssh_key_rotation_sshd_service }}" - state: restarted + ssh_key_rotation_test_keys: + - name: old_key + type: rsa + bits: 3072 + - name: new_key + type: ed25519 + ssh_key_rotation_test_seed_key: old_key + ssh_key_rotation_test_rotation_vars: + old: old_key + new: new_key diff --git a/extensions/molecule/default/verify.yml b/extensions/molecule/default/verify.yml index 0d95b6e..864123b 100644 --- a/extensions/molecule/default/verify.yml +++ b/extensions/molecule/default/verify.yml @@ -3,14 +3,19 @@ # Confirms the rotation actually did what it claims: the new key authenticates, the old key # is rejected, authorized_keys/sshd_config reflect the role's (default) settings. -- name: Verify | Confirm rotation results via the NEW key +- name: Verify | Inspect the post-rotation state on each container hosts: rotate gather_facts: false vars: ssh_key_rotation_test_dir: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}" - ansible_ssh_private_key_file: "{{ ssh_key_rotation_test_dir }}/new_key" tasks: - - name: Verify | New key authenticates + # Deliberately NOT presented as a key-authentication check. This play reaches the + # containers over ansible_connection=community.docker.docker, which ignores + # ansible_ssh_private_key_file entirely, so a ping here returns "pong" no matter which key + # (or no key) is supplied and can prove nothing about SSH auth. That proof is the second + # play below, which drives a real ssh client. This is only a liveness check for the file + # inspection that follows. + - name: Verify | Container is reachable for inspection ansible.builtin.ping: - name: Verify | Read authorized_keys diff --git a/extensions/molecule/default/Dockerfile.j2 b/extensions/molecule/resources/Dockerfile.j2 similarity index 63% rename from extensions/molecule/default/Dockerfile.j2 rename to extensions/molecule/resources/Dockerfile.j2 index 9bfa74d..aed9886 100644 --- a/extensions/molecule/default/Dockerfile.j2 +++ b/extensions/molecule/resources/Dockerfile.j2 @@ -1,8 +1,8 @@ # Molecule managed # -# Wraps the upstream geerlingguy image (already has python/ansible prereqs) with an -# entrypoint that pre-creates root's ansible remote_tmp dir before systemd starts, so the -# first ansible task never races container/systemd boot for that mkdir. +# Shared by every scenario (referenced with `dockerfile: ../resources/Dockerfile.j2` on each +# platform). Wraps the upstream geerlingguy image, which already ships the python/ansible +# prerequisites, with the two things these tests need on top of it. FROM {{ item.image }} # These test images don't ship openssh-server. Install it at build time (not via an Ansible @@ -28,7 +28,10 @@ RUN dnf install -y openssh-server \ RUN chmod 0400 /etc/shadow {% endif %} -COPY molecule-entrypoint.sh /usr/local/bin/molecule-entrypoint.sh -RUN chmod +x /usr/local/bin/molecule-entrypoint.sh - -ENTRYPOINT ["/usr/local/bin/molecule-entrypoint.sh"] +# Pre-create root's ansible remote_tmp directory, the absolute path that molecule.yml pins +# ansible_remote_tmp to for the "rotate" group. Doing it in the image (rather than from an +# entrypoint wrapper or a prepare task) means the directory exists before the container is +# ever started, so nothing at runtime can race the first Ansible connection's own mkdir of +# the same path. /root is neither a tmpfs nor a volume in these platforms, so the layer +# survives into the running container. +RUN install -d -m 0700 /root/.ansible/tmp diff --git a/extensions/molecule/resources/prepare.yml b/extensions/molecule/resources/prepare.yml new file mode 100644 index 0000000..f39b46f --- /dev/null +++ b/extensions/molecule/resources/prepare.yml @@ -0,0 +1,119 @@ +--- + +# Shared by every Molecule scenario: each scenario's prepare.yml is a thin wrapper that +# imports this playbook and supplies the vars listed below. Prepares each Molecule-managed +# container as a realistic rotation target: ephemeral keypairs generated on the control node, +# one of them pre-authorized (simulating existing access before install.yml ever runs), and a +# generated inventory that converge.yml uses to reach the containers over actual SSH (not the +# Molecule docker connection plugin, since that's not what the role uses in production). +# +# Vars each scenario must supply: +# ssh_key_rotation_test_keys List of keypairs to generate, as {name, type, bits}. +# `bits` is optional and only meaningful for RSA. +# ssh_key_rotation_test_seed_key Name of the keypair to install as pre-existing authorized +# access on every container. +# +# Optional: +# ssh_key_rotation_test_rotation_vars +# {old: , new: }. When set, writes +# rotation_vars.yml, the single extra-vars file used by a +# converge that shells out to playbooks/rotate.yml. Scenarios +# that drive the role's stages directly don't need it. + +- name: Prepare | Generate ephemeral keys and the test inventory on the control node + hosts: localhost + connection: local + gather_facts: false + vars: + ssh_key_rotation_test_dir: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}" + tasks: + - name: Prepare | Generate the ephemeral test keypairs + ansible.builtin.command: + cmd: >- + ssh-keygen -t {{ item.type }} + {{ ('-b ' ~ item.bits) if item.bits is defined else '' }} + -N '' -f {{ ssh_key_rotation_test_dir }}/{{ item.name }} + creates: "{{ ssh_key_rotation_test_dir }}/{{ item.name }}" + loop: "{{ ssh_key_rotation_test_keys }}" + loop_control: + label: "{{ item.name }} ({{ item.type }})" + + - name: Prepare | Write the generated rotation vars file used by converge.yml + when: ssh_key_rotation_test_rotation_vars | default({}) | length > 0 + vars: + ssh_key_rotation_test_old: "{{ ssh_key_rotation_test_rotation_vars.old }}" + ssh_key_rotation_test_new: "{{ ssh_key_rotation_test_rotation_vars.new }}" + ansible.builtin.copy: + dest: "{{ ssh_key_rotation_test_dir }}/rotation_vars.yml" + mode: "0600" + content: | + --- + old_private_key: "{{ ssh_key_rotation_test_dir }}/{{ ssh_key_rotation_test_old }}" + old_public_key_file: "{{ ssh_key_rotation_test_dir }}/{{ ssh_key_rotation_test_old }}.pub" + new_private_key: "{{ ssh_key_rotation_test_dir }}/{{ ssh_key_rotation_test_new }}" + new_public_key_file: "{{ ssh_key_rotation_test_dir }}/{{ ssh_key_rotation_test_new }}.pub" + ssh_key_rotation_target_user: root + +- name: Prepare | Seed the pre-existing key into each container's authorized_keys + hosts: rotate + gather_facts: true + vars: + ssh_key_rotation_test_dir: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}" + # openssh-server is installed at Dockerfile build time (see resources/Dockerfile.j2), not + # here. The service/unit name still differs by family: Debian/Ubuntu: ssh, RedHat/Rocky: sshd. + ssh_key_rotation_sshd_service: "{{ 'sshd' if ansible_facts['os_family'] == 'RedHat' else 'ssh' }}" + tasks: + - name: Prepare | Ensure sshd is running + ansible.builtin.systemd: + name: "{{ ssh_key_rotation_sshd_service }}" + state: started + enabled: true + + - name: Prepare | Ensure PermitRootLogin allows key auth (test images default to prohibit-password already) + ansible.builtin.lineinfile: + path: /etc/ssh/sshd_config + regexp: '^#?\s*PermitRootLogin' + line: 'PermitRootLogin prohibit-password' + notify: Restart sshd for prepare + + - name: Prepare | Install the ephemeral seed public key as pre-existing authorized access + ansible.posix.authorized_key: + user: root + key: "{{ lookup('file', ssh_key_rotation_test_dir + '/' + ssh_key_rotation_test_seed_key + '.pub') }}" + state: present + + - name: Prepare | Discover this container's IP address on the docker network + ansible.builtin.command: hostname -I + changed_when: false + register: ssh_key_rotation_container_ip + + - name: Prepare | Record this container's IP for the generated inventory + ansible.builtin.set_fact: + ssh_key_rotation_container_ip: "{{ ssh_key_rotation_container_ip.stdout.split() | first }}" + + - name: Prepare | Ensure the [rotate] group header is present in the generated inventory + ansible.builtin.lineinfile: + path: "{{ ssh_key_rotation_test_dir }}/rotation_inventory.ini" + line: "[rotate]" + insertbefore: BOF + create: true + mode: "0600" + delegate_to: localhost + run_once: true + + - name: Prepare | Append this host to the generated rotation inventory + ansible.builtin.lineinfile: + path: "{{ ssh_key_rotation_test_dir }}/rotation_inventory.ini" + create: true + mode: "0600" + line: >- + {{ inventory_hostname }} ansible_host={{ ssh_key_rotation_container_ip }} ansible_user=root + ansible_ssh_common_args='-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null' + insertafter: EOF + delegate_to: localhost + + handlers: + - name: Restart sshd for prepare + ansible.builtin.systemd: + name: "{{ ssh_key_rotation_sshd_service }}" + state: restarted diff --git a/extensions/molecule/rollback/Dockerfile.j2 b/extensions/molecule/rollback/Dockerfile.j2 deleted file mode 100644 index 9bfa74d..0000000 --- a/extensions/molecule/rollback/Dockerfile.j2 +++ /dev/null @@ -1,34 +0,0 @@ -# Molecule managed -# -# Wraps the upstream geerlingguy image (already has python/ansible prereqs) with an -# entrypoint that pre-creates root's ansible remote_tmp dir before systemd starts, so the -# first ansible task never races container/systemd boot for that mkdir. -FROM {{ item.image }} - -# These test images don't ship openssh-server. Install it at build time (not via an Ansible -# task in prepare.yml) so any slowness/hang shows up in the docker build log instead of being -# buried in Ansible task output. -{% if 'ubuntu' in item.image or 'debian' in item.image %} -RUN apt-get update \ - && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends openssh-server \ - && rm -rf /var/lib/apt/lists/* -{% else %} -RUN dnf install -y openssh-server \ - && dnf clean all - -# rockylinux9 ships /etc/shadow as mode 0000 (the RHEL default) but with a pam old enough -# (1.5.1-28.el9) that pam_unix's ACCOUNT phase resolves shadow via the unix_chkpwd helper. -# That helper links libcap-ng and deliberately drops its capabilities, so it loses -# DAC_OVERRIDE and cannot open a 0000 file even as uid 0 - the lookup fails with -# PAM_AUTHINFO_UNAVAIL. sshd (UsePAM yes) then rejects every login with "Access denied for -# user root by PAM account configuration", which the client sees as a bare -# "Connection closed by port 22"; sudo fails the same way. rockylinux10's pam 1.6.1 -# does not use the helper here, which is why only rockylinux9 is affected. -# 0400 root:root is still root-only, so this grants no access that 0000 didn't already imply. -RUN chmod 0400 /etc/shadow -{% endif %} - -COPY molecule-entrypoint.sh /usr/local/bin/molecule-entrypoint.sh -RUN chmod +x /usr/local/bin/molecule-entrypoint.sh - -ENTRYPOINT ["/usr/local/bin/molecule-entrypoint.sh"] diff --git a/extensions/molecule/rollback/molecule-entrypoint.sh b/extensions/molecule/rollback/molecule-entrypoint.sh deleted file mode 100755 index d0a681e..0000000 --- a/extensions/molecule/rollback/molecule-entrypoint.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash -set -euo pipefail - -# Pre-create root's ansible remote_tmp dir before handing off to the real container command -# (systemd via /usr/sbin/init). This runs synchronously before the container is reported as -# started, so it can never race the first ansible connection's own mkdir of this same path. -install -d -m 0700 /root/.ansible/tmp - -exec "$@" diff --git a/extensions/molecule/rollback/molecule.yml b/extensions/molecule/rollback/molecule.yml index 0b140b3..ef25da5 100644 --- a/extensions/molecule/rollback/molecule.yml +++ b/extensions/molecule/rollback/molecule.yml @@ -3,8 +3,13 @@ driver: name: docker platforms: - - name: ssh-key-rotation-rollback-ubuntu2204 + # The first platform carries the settings every platform shares; the rest merge it and + # override only name/image. The Dockerfile itself is shared across scenarios too, hence the + # relative path (the docker driver resolves `dockerfile` against this scenario directory). + - &platform_defaults + name: ssh-key-rotation-rollback-ubuntu2204 image: geerlingguy/docker-ubuntu2204-ansible:latest + dockerfile: ../resources/Dockerfile.j2 command: /usr/sbin/init privileged: true cgroupns_mode: host @@ -16,43 +21,31 @@ platforms: - /sys/fs/cgroup:/sys/fs/cgroup:rw groups: - rotate - - name: ssh-key-rotation-rollback-rockylinux9 + - <<: *platform_defaults + name: ssh-key-rotation-rollback-rockylinux9 image: geerlingguy/docker-rockylinux9-ansible:latest - command: /usr/sbin/init - privileged: true - cgroupns_mode: host - pre_build_image: false - tmpfs: - - /run - - /run/lock - volumes: - - /sys/fs/cgroup:/sys/fs/cgroup:rw - groups: - - rotate - - name: ssh-key-rotation-rollback-rockylinux10 + - <<: *platform_defaults + name: ssh-key-rotation-rollback-rockylinux10 image: geerlingguy/docker-rockylinux10-ansible:latest - command: /usr/sbin/init - privileged: true - cgroupns_mode: host - pre_build_image: false - tmpfs: - - /run - - /run/lock - volumes: - - /sys/fs/cgroup:/sys/fs/cgroup:rw - groups: - - rotate +dependency: + # Mirrors exactly what CI used to run as a separate step, so a local `molecule test` and a + # CI run install the same collection dependencies. Molecule chdirs to the project directory + # (the collection root) before running a sequence, so this relative path is correct. + name: shell + command: ansible-galaxy install -r requirements.yml provisioner: name: ansible - env: - ANSIBLE_ROLES_PATH: "../../../roles" + # Deliberately no ANSIBLE_ROLES_PATH here. Molecule runs with the project directory as its + # cwd, not the scenario directory, so a scenario-relative value resolves outside the repo and, + # worse, overrides the roles_path in the collection's own ansible.cfg - which is what makes the + # bare `ssh_key_rotation` role name in converge.yml resolve at all. inventory: group_vars: # The docker connection plugin doesn't invoke a login shell, so `~` in the default # remote_tmp path doesn't reliably expand for the "rotate" containers. Pin it to the - # absolute path molecule-entrypoint.sh already pre-creates for root before systemd - # starts. Scoped to this group only - localhost (used by molecule's own internal - # playbooks and the key-generation play) must keep its normal, non-root remote_tmp. + # absolute path resources/Dockerfile.j2 already creates for root in the image. Scoped to + # this group only - localhost (used by molecule's own internal playbooks and the + # key-generation play) must keep its normal, non-root remote_tmp. rotate: ansible_remote_tmp: /root/.ansible/tmp # These containers are entered as root. Saying so explicitly means Ansible skips @@ -64,6 +57,11 @@ provisioner: ansible_user: root scenario: test_sequence: + - dependency + # Leading destroy: without it a container left behind by an aborted run is reused, and its + # authorized_keys already holds keys from that run. That silently breaks this suite's core + # premise (only the seeded key is authorized before converge runs). + - destroy - syntax - create - prepare diff --git a/extensions/molecule/rollback/prepare.yml b/extensions/molecule/rollback/prepare.yml index 96df43c..3220d56 100644 --- a/extensions/molecule/rollback/prepare.yml +++ b/extensions/molecule/rollback/prepare.yml @@ -1,91 +1,21 @@ --- -# Same environment setup as molecule/default: ephemeral keys + a pre-authorized "old" key + -# a generated inventory. This scenario needs THREE keypairs: seed_key (pre-existing access), -# key_a (installed by a first, successful rotation used as the "known-good" checkpoint), and -# key_b (the second rotation, deliberately broken partway through verify, to prove the -# rescue block restores access to key_a). - -- name: Prepare | Generate ephemeral keys and the test inventory on the control node - hosts: localhost - connection: local - gather_facts: false - vars: - ssh_key_rotation_test_dir: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}" - tasks: - - name: Prepare | Generate the ephemeral SEED (pre-existing) keypair - ansible.builtin.command: "ssh-keygen -t rsa -b 3072 -N '' -f {{ ssh_key_rotation_test_dir }}/seed_key" - args: - creates: "{{ ssh_key_rotation_test_dir }}/seed_key" - - - name: Prepare | Generate the ephemeral KEY_A keypair (first, successful rotation) - ansible.builtin.command: "ssh-keygen -t ed25519 -N '' -f {{ ssh_key_rotation_test_dir }}/key_a" - args: - creates: "{{ ssh_key_rotation_test_dir }}/key_a" - - - name: Prepare | Generate the ephemeral KEY_B keypair (second, deliberately broken rotation) - ansible.builtin.command: "ssh-keygen -t ed25519 -N '' -f {{ ssh_key_rotation_test_dir }}/key_b" - args: - creates: "{{ ssh_key_rotation_test_dir }}/key_b" - -- name: Prepare | Seed the SEED key into each container's authorized_keys - hosts: rotate - gather_facts: true +# This scenario needs THREE keypairs: seed_key (pre-existing access), key_a (installed by a +# first, successful rotation used as the "known-good" checkpoint), and key_b (the second +# rotation, deliberately broken partway through verify, to prove the rescue block restores +# access to key_a). It invokes the role's stages directly rather than shelling out to +# playbooks/rotate.yml, so it needs no generated rotation_vars.yml - converge.yml sets the +# per-rotation vars itself. Everything else is in resources/prepare.yml. + +- name: Prepare | Set up the containers as rotation targets + ansible.builtin.import_playbook: ../resources/prepare.yml vars: - ssh_key_rotation_test_dir: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}" - # openssh-server is installed at Dockerfile build time (see Dockerfile.j2), not here. The - # service/unit name still differs by family: Debian/Ubuntu: ssh, RedHat/Rocky: sshd. - ssh_key_rotation_sshd_service: "{{ 'sshd' if ansible_facts['os_family'] == 'RedHat' else 'ssh' }}" - tasks: - - name: Prepare | Ensure sshd is running - ansible.builtin.systemd: - name: "{{ ssh_key_rotation_sshd_service }}" - state: started - enabled: true - - - name: Prepare | Ensure PermitRootLogin allows key auth - ansible.builtin.lineinfile: - path: /etc/ssh/sshd_config - regexp: '^#?\s*PermitRootLogin' - line: 'PermitRootLogin prohibit-password' - notify: Restart sshd for prepare - - - name: Prepare | Install the ephemeral SEED public key as pre-existing authorized access - ansible.posix.authorized_key: - user: root - key: "{{ lookup('file', ssh_key_rotation_test_dir + '/seed_key.pub') }}" - state: present - - - name: Prepare | Discover this container's IP address on the docker network - ansible.builtin.command: hostname -I - changed_when: false - register: ssh_key_rotation_container_ip - - - name: Prepare | Record this container's IP for the generated inventory - ansible.builtin.set_fact: - ssh_key_rotation_container_ip: "{{ ssh_key_rotation_container_ip.stdout.split() | first }}" - - - name: Prepare | Ensure the [rotate] group header is present in the generated inventory - ansible.builtin.lineinfile: - path: "{{ ssh_key_rotation_test_dir }}/rotation_inventory.ini" - line: "[rotate]" - insertbefore: BOF - create: true - mode: "0600" - delegate_to: localhost - run_once: true - - - name: Prepare | Append this host to the generated rotation inventory - ansible.builtin.lineinfile: - path: "{{ ssh_key_rotation_test_dir }}/rotation_inventory.ini" - line: >- - {{ inventory_hostname }} ansible_host={{ ssh_key_rotation_container_ip }} ansible_user=root - ansible_ssh_common_args='-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null' - insertafter: EOF - delegate_to: localhost - - handlers: - - name: Restart sshd for prepare - ansible.builtin.systemd: - name: "{{ ssh_key_rotation_sshd_service }}" - state: restarted + ssh_key_rotation_test_keys: + - name: seed_key + type: rsa + bits: 3072 + - name: key_a + type: ed25519 + - name: key_b + type: ed25519 + ssh_key_rotation_test_seed_key: seed_key diff --git a/extensions/molecule/rollback/verify.yml b/extensions/molecule/rollback/verify.yml index 1e05c7b..881d197 100644 --- a/extensions/molecule/rollback/verify.yml +++ b/extensions/molecule/rollback/verify.yml @@ -6,9 +6,13 @@ - name: Verify | Confirm key_a still authenticates after the failed/rolled-back rotation hosts: rotate - gather_facts: false + # Facts are needed for the family-specific sshd unit name below. + gather_facts: true vars: ssh_key_rotation_test_dir: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}" + # Same lookup prepare.yml uses: Debian/Ubuntu: ssh, RedHat/Rocky: sshd. Hardcoding "sshd" + # here only worked on Ubuntu by accident, via ssh.service's sshd.service alias. + ssh_key_rotation_sshd_service: "{{ 'sshd' if ansible_facts['os_family'] == 'RedHat' else 'ssh' }}" ssh_key_rotation_verify_ssh_opts: >- -o BatchMode=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o IdentitiesOnly=yes -o ConnectTimeout=15 @@ -31,7 +35,7 @@ changed_when: false - name: Verify | sshd is running - ansible.builtin.command: systemctl is-active sshd + ansible.builtin.command: "systemctl is-active {{ ssh_key_rotation_sshd_service }}" become: true changed_when: false From dd9d2eb07594ad3ad1dea8ebe76cc2341492c46c Mon Sep 17 00:00:00 2001 From: Mark Bolwell Date: Fri, 31 Jul 2026 02:26:16 +0100 Subject: [PATCH 04/18] Fixed rescue with path issue --- CHANGELOG.md | 2 +- extensions/molecule/default/molecule.yml | 16 +++++++++++----- extensions/molecule/rollback/molecule.yml | 13 +++++++++---- 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c29dbe..34ea22e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ - Molecule idempotency check required `changed=0` from the timestamped backup tasks, which cannot be idempotent by design; it now allows those and fails on anything else. - Molecule rollback check compared `authorized_keys` against a checkpoint taken before the install stage, demanding the rescue undo work it never performs. - Molecule rollback "sshd is running" check hardcoded the `sshd` unit, which resolves on Ubuntu only through an alias; it now uses the same family lookup as prepare. -- Molecule set `ANSIBLE_ROLES_PATH` to a scenario-relative path, but Molecule runs with the project directory as its cwd, so it resolved outside the repo and overrode the `roles_path` in `ansible.cfg` that makes a bare `ssh_key_rotation` role name resolve. Removed, so `ansible.cfg` applies. +- Molecule set `ANSIBLE_ROLES_PATH` to a scenario-relative path (`../../../roles`), but Molecule runs with the project directory as its cwd, so it resolved outside the repo entirely and the rollback scenario could not find the role. The collection's own `ansible.cfg` cannot cover for it either, since Molecule generates its own `ansible.cfg` and points `ANSIBLE_CONFIG` at it. It is now `${MOLECULE_PROJECT_DIRECTORY:-.}/roles`, which is the collection root in both cases. - The default scenario's first verify play was named as a new-key authentication check but runs over the container connection, which ignores SSH keys, so it could never fail; it is now named and documented as the reachability check it actually is. The key-authentication proof was already in the following play, which drives a real ssh client. ### Changed diff --git a/extensions/molecule/default/molecule.yml b/extensions/molecule/default/molecule.yml index 899f74c..f3977c9 100644 --- a/extensions/molecule/default/molecule.yml +++ b/extensions/molecule/default/molecule.yml @@ -35,11 +35,17 @@ dependency: command: ansible-galaxy install -r requirements.yml provisioner: name: ansible - # Deliberately no ANSIBLE_ROLES_PATH here. Molecule runs with the project directory as its - # cwd, not the scenario directory, so a scenario-relative value resolves outside the repo and, - # worse, overrides the roles_path in the collection's own ansible.cfg - which is what makes a - # bare `ssh_key_rotation` role name resolve. Leaving it unset lets ansible.cfg do its job. - # The converge that shells out to playbooks/rotate.yml sets its own absolute path. + env: + # This scenario's converge shells out to playbooks/rotate.yml with its own absolute roles + # path, so nothing here strictly needs this today. It is set anyway to keep both scenarios + # identical and to stay correct if a play here ever includes the role directly. Two traps: + # the collection's own ansible.cfg (roles_path = ./roles) is NOT consulted, because + # Molecule generates its own ansible.cfg and points ANSIBLE_CONFIG at it; and Molecule + # runs with the project directory as cwd, not the scenario directory, so a + # scenario-relative path like ../../../roles resolves outside the repo entirely. + # MOLECULE_PROJECT_DIRECTORY is the collection root, and the `.` fallback is the same + # thing whenever it isn't exported, since Molecule chdirs there before running a sequence. + ANSIBLE_ROLES_PATH: "${MOLECULE_PROJECT_DIRECTORY:-.}/roles" inventory: group_vars: # The docker connection plugin doesn't invoke a login shell, so `~` in the default diff --git a/extensions/molecule/rollback/molecule.yml b/extensions/molecule/rollback/molecule.yml index ef25da5..c29efb6 100644 --- a/extensions/molecule/rollback/molecule.yml +++ b/extensions/molecule/rollback/molecule.yml @@ -35,10 +35,15 @@ dependency: command: ansible-galaxy install -r requirements.yml provisioner: name: ansible - # Deliberately no ANSIBLE_ROLES_PATH here. Molecule runs with the project directory as its - # cwd, not the scenario directory, so a scenario-relative value resolves outside the repo and, - # worse, overrides the roles_path in the collection's own ansible.cfg - which is what makes the - # bare `ssh_key_rotation` role name in converge.yml resolve at all. + env: + # converge.yml includes the role by its bare name, so this has to be right. Two traps: + # the collection's own ansible.cfg (roles_path = ./roles) is NOT consulted, because + # Molecule generates its own ansible.cfg and points ANSIBLE_CONFIG at it; and Molecule + # runs with the project directory as cwd, not the scenario directory, so a + # scenario-relative path like ../../../roles resolves outside the repo entirely. + # MOLECULE_PROJECT_DIRECTORY is the collection root, and the `.` fallback is the same + # thing whenever it isn't exported, since Molecule chdirs there before running a sequence. + ANSIBLE_ROLES_PATH: "${MOLECULE_PROJECT_DIRECTORY:-.}/roles" inventory: group_vars: # The docker connection plugin doesn't invoke a login shell, so `~` in the default From 3098ecb3205137e3fc021644e7c5b4bd05456e88 Mon Sep 17 00:00:00 2001 From: Mark Bolwell Date: Tue, 4 Aug 2026 12:20:54 +0100 Subject: [PATCH 05/18] Added galaxy release setup Signed-off-by: Mark Bolwell --- .github/workflows/release.yml | 57 +++++++++++++++++++++++++++++++++++ .gitignore | 4 +++ CHANGELOG.md | 11 +++++++ DEVELOPMENT.md | 23 +++++++++++--- galaxy.yml | 18 ++++++----- 5 files changed, 100 insertions(+), 13 deletions(-) create mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..419d0d5 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,57 @@ +--- + +name: Release to Ansible Galaxy + +on: + push: + tags: + - "v*" + # Lets the build be exercised without cutting a tag; publishing stays gated on the tag. + workflow_dispatch: + +jobs: + build-and-publish: + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v7 + + - name: Set up Python + uses: actions/setup-python@v7 + with: + python-version: "3.12" + + - name: Install Ansible + run: | + python -m pip install --upgrade pip + pip install "ansible>=2.18" + + # A tag of v1.2.3 must match `version: 1.2.3` in galaxy.yml. Galaxy rejects a re-upload of + # an existing version, so catching the mismatch here avoids a half-finished release. + - name: Check the tag matches galaxy.yml + if: startsWith(github.ref, 'refs/tags/v') + run: | + tag_version="${GITHUB_REF_NAME#v}" + galaxy_version="$(python -c "import yaml,sys; print(yaml.safe_load(open('galaxy.yml'))['version'])")" + if [ "$tag_version" != "$galaxy_version" ]; then + echo "Tag ${GITHUB_REF_NAME} does not match galaxy.yml version ${galaxy_version}" >&2 + exit 1 + fi + + - name: Build the collection artifact + run: ansible-galaxy collection build --output-path ./dist + + - name: Upload the artifact to the workflow run + uses: actions/upload-artifact@v5 + with: + name: collection-tarball + path: dist/*.tar.gz + + - name: Publish to Ansible Galaxy + if: startsWith(github.ref, 'refs/tags/v') + env: + GALAXY_API_KEY: ${{ secrets.GALAXY_API_KEY }} + run: | + ansible-galaxy collection publish \ + dist/krameff-ssh_key_rotation-*.tar.gz \ + --api-key "$GALAXY_API_KEY" diff --git a/.gitignore b/.gitignore index f0b696f..e48cc38 100644 --- a/.gitignore +++ b/.gitignore @@ -39,3 +39,7 @@ pwc_id_* *.pem authorized_keys.bak-* test_vars.yml +test_* +*_ecdsa* +*_ed25519* +!*.pub.example diff --git a/CHANGELOG.md b/CHANGELOG.md index 34ea22e..42f24af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ ## [Unreleased] +### Added + +- Release workflow: tagging `v*` builds the collection and publishes it to Galaxy, needs a `GALAXY_API_KEY` secret. +- Release workflow fails the build if the tag doesn't match the version in `galaxy.yml`. + ### Fixed - Verify stage rollback no longer aborts if the sshd reload fails - it finishes restoring, re-confirms connectivity, and reports that sshd needs a manual reload. @@ -23,6 +28,12 @@ - Molecule platform definitions use a YAML anchor rather than repeating three near-identical blocks per scenario. - Molecule test sequences start with `dependency` and `destroy`: collection dependencies are installed the same way locally and in CI (the workflow's separate `ansible-galaxy` step is gone), and a container left behind by an aborted run can no longer be reused with keys already in its `authorized_keys`. - The Molecule workflow also runs weekly, so drift in the `:latest` test images surfaces on its own build rather than on an unrelated pull request. +- `build_ignore` still pointed at `molecule/`, so the scenarios were being shipped in the artifact; it now excludes `extensions/molecule` plus `.github`, `.claude`, `.cursor`, `.ansible`, `.mcp.json` and `inventory.ini`. +- `DEVELOPMENT.md` shows how to generate throwaway keys outside the repo for manual testing. + +### Removed + +- Root-level test keypairs and `test_vars.yml` - local scratch files nothing depended on. ## [Initial] diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 011796d..1b1192e 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -151,11 +151,24 @@ molecule test -s rollback ``` Both scenarios generate their own ephemeral SSH keypairs and inventory into Molecule's -per-run ephemeral directory - they do not use the root-level `test_vars.yml` or -`test_rsa*`/`test_ecdsa*`/`test_ed25519` files. Those root-level files are for local, -manual testing only (e.g. `ansible-playbook -i inventory.ini playbooks/rotate.yml -e -@test_vars.yml`); edit `test_vars.yml`'s paths for your own machine before using it, since -the checked-in paths are just examples. +per-run ephemeral directory, so nothing needs to exist in the repository for them to run. + +For local manual testing, generate your own throwaway keypairs outside the repository and +point the playbook at them: + +```bash +ssh-keygen -t rsa -b 4096 -N "" -f /tmp/rotation-old +ssh-keygen -t ed25519 -N "" -f /tmp/rotation-new + +ansible-playbook -i inventory.ini playbooks/rotate.yml \ + -e old_private_key=/tmp/rotation-old \ + -e old_public_key_file=/tmp/rotation-old.pub \ + -e new_private_key=/tmp/rotation-new \ + -e new_public_key_file=/tmp/rotation-new.pub +``` + +Keep generated keys out of the working tree. `.gitignore` covers the common patterns, but +a private key that lands in a commit has to be treated as compromised regardless. ## Building and Publishing diff --git a/galaxy.yml b/galaxy.yml index 9063b43..69d3c2b 100644 --- a/galaxy.yml +++ b/galaxy.yml @@ -20,11 +20,13 @@ tags: dependencies: ansible.posix: ">=1.3.0" build_ignore: - - molecule - - test_vars.yml - - test_rsa - - test_rsa.pub - - test_ecdsa - - test_ecdsa.pub - - test_ed25519 - - test_ed25519.pub + - extensions/molecule + - .github + - .claude + - .cursor + - .ansible + - .mcp.json + - inventory.ini + # Belt and braces: these are gitignored and should never exist in a clean checkout, but a + # build run from a developer's working tree must not pick up local key material. + - test_* From f6955945eca16e24e2bfad75a310228bf8d0cf85 Mon Sep 17 00:00:00 2001 From: Mark Bolwell Date: Wed, 5 Aug 2026 11:19:23 +0100 Subject: [PATCH 06/18] Add core files --- .pre-commit-config.yaml | 68 +++++++++++++++++++++++++++++++++++++++++ .yamllint | 39 +++++++++++++++++++++++ 2 files changed, 107 insertions(+) create mode 100644 .pre-commit-config.yaml create mode 100644 .yamllint diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..fa247a9 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,68 @@ +--- + +##### CI for use by github no need for action to be added +##### Inherited +ci: + autofix_prs: false + skip: [detect-aws-credentials, ansible-lint] + +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v6.0.0 + hooks: + # Safety + - id: detect-aws-credentials + name: Detect AWS Credentials + - id: detect-private-key + name: Detect Private Keys + + # git checks + - id: check-merge-conflict + name: Check for merge conflicts + - id: check-added-large-files + name: Check for Large files + - id: check-case-conflict + name: Check case conflict + + # General checks + - id: trailing-whitespace + name: Trim Trailing Whitespace + description: This hook trims trailing whitespace. + entry: trailing-whitespace-fixer + language: python + types: [text] + args: [--markdown-linebreak-ext=md] + - id: end-of-file-fixer + name: Ensure line at end of file + +# Scan for passwords +- repo: https://github.com/Yelp/detect-secrets + rev: v1.5.0 + hooks: + - id: detect-secrets + name: Detect Secrets test + +- repo: https://github.com/gitleaks/gitleaks + rev: v8.30.1 + hooks: + - id: gitleaks + name: Run Gitleaks test + +- repo: https://github.com/ansible-community/ansible-lint + rev: v26.3.0 + hooks: + - id: ansible-lint + name: Ansible-lint + description: This hook runs ansible-lint. + entry: python3 -m ansiblelint --force-color site.yml -c .ansible-lint + language: python + # do not pass files to ansible-lint, see: + # https://github.com/ansible/ansible-lint/issues/611 + pass_filenames: false + always_run: true + +- repo: https://github.com/adrienverge/yamllint.git + rev: v1.38.0 # or higher tag + hooks: + - id: yamllint + name: Check YAML Lint diff --git a/.yamllint b/.yamllint new file mode 100644 index 0000000..c1b1254 --- /dev/null +++ b/.yamllint @@ -0,0 +1,39 @@ +--- + +extends: default +ignore: | + tests/ + molecule/ + .github/ + .gitlab-ci.yml + *molecule.yml +rules: + braces: + max-spaces-inside: 1 + level: error + brackets: + max-spaces-inside: 1 + level: error + comments: + ignore-shebangs: true + min-spaces-from-content: 1 # prettier compatibility + comments-indentation: disable + empty-lines: + max: 1 + indentation: + # Requiring 2 space indentation + spaces: 2 + # Requiring consistent indentation within a file, either indented or not + indent-sequences: consistent + key-duplicates: enable + line-length: disable + new-line-at-end-of-file: enable + new-lines: + type: unix + octal-values: + forbid-implicit-octal: true # yamllint defaults to false + forbid-explicit-octal: true + trailing-spaces: enable + truthy: + allowed-values: ['true', 'false'] + check-keys: true From 3b2a5dc2b5b1de8ae55a01c92e0469925aa3df66 Mon Sep 17 00:00:00 2001 From: Mark Bolwell Date: Wed, 5 Aug 2026 11:19:41 +0100 Subject: [PATCH 07/18] Issues and bug reports --- .github/ISSUE_TEMPLATE/bug_report.yml | 130 +++++++++++++++++++++ .github/ISSUE_TEMPLATE/config.yml | 17 +++ .github/ISSUE_TEMPLATE/feature_request.yml | 51 ++++++++ .github/pull_request_template.md | 51 ++++++++ 4 files changed, 249 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.yml create mode 100644 .github/ISSUE_TEMPLATE/config.yml create mode 100644 .github/ISSUE_TEMPLATE/feature_request.yml create mode 100644 .github/pull_request_template.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 0000000..d80c8e3 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,130 @@ +--- + +name: Bug report +description: Something went wrong during a rotation +labels: [bug] +body: + - type: markdown + attributes: + value: >- + Before filing, please check the [troubleshooting + section](https://github.com/krameff/ssh_key_rotation#troubleshooting) of + the README. It covers the common failure modes, including crypto-policy + rejections and `sshd_config.d` drop-ins overriding what the playbook + writes. + + + If you have found a way this collection can lock a host out, please + report it privately instead. See the security link on the previous page. + + - type: textarea + id: what-happened + attributes: + label: What happened + description: What did you expect, and what did you get instead? + validations: + required: true + + - type: dropdown + id: phase + attributes: + label: Which phase failed + options: + - "Phase 0: validate, on the control node" + - "Phase 1: install, connected with the old key" + - "Phase 2: verify, reconnecting with the new key" + - Not sure + validations: + required: true + + - type: dropdown + id: host-state + attributes: + label: Was the host left reachable + description: >- + This is the most important field. If a run left a host unreachable, that + is a safety model failure and will be prioritised over everything else. + options: + - "Yes, the old key still works" + - "Yes, the new key works" + - "No, the host is unreachable" + - Not sure yet + validations: + required: true + + - type: textarea + id: command + attributes: + label: The command you ran + description: Redact key paths if they are sensitive. + render: shell + validations: + required: true + + - type: textarea + id: output + attributes: + label: Failing task and error output + description: >- + The failing task name and its error. Output from a run with `-vvv` is + much more useful. Please redact any key material. + render: text + validations: + required: true + + - type: input + id: target-os + attributes: + label: Target OS and version + placeholder: "AlmaLinux 9.8, Ubuntu 22.04, openSUSE Leap 15.6" + validations: + required: true + + - type: input + id: ansible-version + attributes: + label: Ansible version + description: Output of `ansible --version` on the control node. + placeholder: "ansible [core 2.18.1]" + validations: + required: true + + - type: input + id: collection-version + attributes: + label: Collection version + placeholder: "1.0.0, or a commit SHA if installed from source" + validations: + required: true + + - type: input + id: key-types + attributes: + label: Old and new key types + placeholder: "old rsa 4096, new ed25519" + + - type: input + id: crypto-policy + attributes: + label: Crypto-policy on the target, if RHEL or Fedora family + description: Output of `update-crypto-policies --show`. + placeholder: "FIPS, DEFAULT:PQ, or leave blank if not applicable" + + - type: textarea + id: dropins + attributes: + label: sshd_config drop-in files on the target + description: >- + Output of `ls /etc/ssh/sshd_config.d/`. These frequently override what + the playbook writes. + render: text + + - type: checkboxes + id: confirmations + attributes: + label: Before submitting + options: + - label: I have read the troubleshooting section of the README + required: true + - label: I have redacted all private key material from this report + required: true diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..32e721e --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,17 @@ +--- + +blank_issues_enabled: false + +contact_links: + - name: Lockout or other security issue + url: https://github.com/krameff/ssh_key_rotation/security/advisories/new + about: >- + If you have found a way this collection can lock a host out, or any other + security issue, please report it privately rather than opening a public + issue. You can also email security@krameff.com. + + - name: Question about usage + url: https://github.com/krameff/ssh_key_rotation/discussions + about: >- + For "how do I" questions, start a discussion. Check the README + troubleshooting section first, it covers the common failure modes. diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 0000000..71412bd --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,51 @@ +--- + +name: Feature request +description: Suggest a capability or change +labels: [enhancement] +body: + - type: markdown + attributes: + value: >- + Please check the "Future enhancements" section of + [DEVELOPMENT.md](https://github.com/krameff/ssh_key_rotation/blob/main/DEVELOPMENT.md) + first, in case it is already on the list. + + - type: textarea + id: problem + attributes: + label: What problem would this solve + description: >- + Describe the situation you are actually in, not just the feature you + have in mind. It often turns out there is a better fit than the one + first thought of. + validations: + required: true + + - type: textarea + id: proposal + attributes: + label: What you would like it to do + validations: + required: true + + - type: textarea + id: safety + attributes: + label: Effect on the safety model + description: >- + Would this change when the old key is removed, when `sshd_config` is + validated or backed up, or how the verify stage's rollback works? See + the [safety + model](https://github.com/krameff/ssh_key_rotation#safety-model). Say so + plainly if it would; that is not automatically a blocker, but it needs + discussing up front. + placeholder: "No effect, this is additive and off by default." + validations: + required: true + + - type: textarea + id: alternatives + attributes: + label: Alternatives you have considered + description: Including whether an existing variable already gets you close. diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..892f6eb --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,51 @@ + + +## What this changes + + + +## Effect on the safety model + + + +## How this was tested + + + +## Checklist + +- [ ] `ansible-lint` passes with no new warnings +- [ ] `ansible-playbook playbooks/rotate.yml --syntax-check` passes +- [ ] `molecule test -s default` passes +- [ ] `molecule test -s rollback` passes +- [ ] New behaviour is covered by a test that fails without the change +- [ ] The safety model is preserved, or the change to it is called out above +- [ ] README.md updated for user-facing changes +- [ ] CHANGELOG.md updated under `## [Unreleased]` +- [ ] `galaxy.yml` version is unchanged; releases are cut separately by tag +- [ ] No new dependencies without justification +- [ ] Backwards compatible, or the breaking change is documented +- [ ] No key material, private or public, committed + + From 503456d07531a7791ff71af362421206ea57e883 Mon Sep 17 00:00:00 2001 From: Mark Bolwell Date: Wed, 5 Aug 2026 11:19:58 +0100 Subject: [PATCH 08/18] Fles updated --- CONTRIBUTING.md | 176 +++++++++ DEVELOPMENT.md | 268 ++++++-------- README.md | 451 +++++++++++++---------- SECURITY.md | 29 ++ images/ssh-key-rotation-lockup-dark.svg | 15 + images/ssh-key-rotation-lockup-light.svg | 15 + 6 files changed, 612 insertions(+), 342 deletions(-) create mode 100644 CONTRIBUTING.md create mode 100644 SECURITY.md create mode 100644 images/ssh-key-rotation-lockup-dark.svg create mode 100644 images/ssh-key-rotation-lockup-light.svg diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..69dc2d5 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,176 @@ +# Contributing + +Thanks for your interest in improving this collection. This document covers how to get set up, what is expected of a change, and how to get it reviewed. + +For deeper detail on project layout, adding modules or filters, and publishing releases, see [DEVELOPMENT.md](DEVELOPMENT.md). + +NOTE: due to the number of open-source projects we work with and amount of PRS etc that have not been properly scoped. We only accept PRs from contributors. We will take all the information from issues and bug reports. +If you are interested in becoming a contributor, please drop us an email at . + +## The one rule that matters most + +This collection rotates SSH keys on live hosts. A bug here does not produce a wrong answer, it produces a machine nobody can log into. + +Every change must preserve the safety model described in the [README](README.md#safety-model). In particular: + +- The old key is never removed until the new key has been proven to authenticate over a fresh connection. +- Every `sshd_config` edit is validated with `sshd -t` before it is written. +- `sshd_config` and `authorized_keys` are backed up before every edit. +- Configuration is applied with a reload, never a restart. +- Phase 2's cleanup stays inside the `block`/`rescue` that restores from those backups. + +If your change touches any of those, say so explicitly in the pull request description and explain why it is still safe. A change that weakens a safety gate will not be merged, however convenient it is. + +## Getting set up + +```bash +git clone https://github.com/krameff/ssh_key_rotation +cd ssh_key_rotation + +python -m venv .venv && source .venv/bin/activate +pip install "ansible>=2.18" ansible-lint "molecule>=25.0" "molecule-plugins[docker]" + +ansible-galaxy install -r requirements.yml +``` + +Molecule needs a working Docker or Podman socket. The scenarios pull `geerlingguy` CI images for Ubuntu 22.04, Rocky Linux 9 and Rocky Linux 10. + +## Making a change + +1. Branch from `main`. +2. Make the change, keeping commits focused. One logical change per commit. +3. Run the checks below and make sure they pass. +4. Update the documentation and `CHANGELOG.md`. +5. Open a pull request against `main`. + +### Where things live + +| You are changing | Edit | +|------------------|------| +| Pre-flight validation, before any host is touched | `roles/ssh_key_rotation/tasks/validate.yml` | +| Installing the new key, preparing sshd | `roles/ssh_key_rotation/tasks/install.yml` | +| Proving the new key, removing the old one | `roles/ssh_key_rotation/tasks/verify.yml` | +| RHEL/Fedora crypto-policy handling | `roles/ssh_key_rotation/tasks/manage_crypto_policy.yml` | +| A variable's default value | `roles/ssh_key_rotation/defaults/main.yml` | +| How the three stages are wired together | `playbooks/rotate.yml` | +| Functional tests | `extensions/molecule/default/`, `extensions/molecule/rollback/` | + +The role has no working `tasks/main.yml` entry point by design, because each stage authenticates differently. Do not add one. + +## Running the checks + +These are the same checks CI runs, so running them locally saves a round trip. + +```bash +# Lint the whole repo, not just the playbook, so the role is covered too +ansible-lint + +# Syntax check the entry point +ansible-playbook playbooks/rotate.yml --syntax-check +``` + +`ansible-lint` runs on the `production` profile. Do not add `# noqa` comments to silence a rule without explaining why in the same commit. + +### Functional tests + +```bash +# Full rotation across three OS images, run twice to prove idempotency +molecule test -s default + +# Deliberately breaks a rotation mid-verify, to prove the rescue block restores access +molecule test -s rollback +``` + +Both scenarios generate their own ephemeral SSH keypairs and inventory into Molecule's per-run directory, so nothing needs to exist in your working tree for them to run. + +If you are iterating and do not want the container torn down between runs: + +```bash +molecule converge -s default # run the playbook against live containers +molecule verify -s default # run just the assertions +molecule destroy -s default # clean up when you are done +``` + +### Writing tests + +Any change to the rotation logic needs a test that would fail without it. + +Two things to watch out for, both of which have produced tests that could never fail: + +- **The container connection ignores SSH keys.** A task run over Molecule's `docker`/`podman` connection proves reachability, not authentication. To test that a key actually works, drive a real `ssh` client. +- **Match key material literally, not as a regex.** Base64 key data contains characters that are regex metacharacters, so a naive pattern can match when the key is still present. + +Before you rely on a new assertion, break the thing it checks and confirm the test goes red. + +## Manual testing against a real host + +Generate throwaway keys outside the repository: + +```bash +ssh-keygen -t rsa -b 4096 -N "" -f /tmp/rotation-old +ssh-keygen -t ed25519 -N "" -f /tmp/rotation-new + +ansible-playbook -i inventory.ini playbooks/rotate.yml \ + -e old_private_key=/tmp/rotation-old \ + -e old_public_key_file=/tmp/rotation-old.pub \ + -e new_private_key=/tmp/rotation-new \ + -e new_public_key_file=/tmp/rotation-new.pub +``` + +Use a disposable VM you can snapshot and roll back. Never test against a host you care about. + +Keep generated keys out of the working tree. `.gitignore` covers the common patterns, but a private key that lands in a commit has to be treated as compromised regardless of whether the commit was pushed. + +## Documentation + +Documentation is part of the change, not a follow-up. + +- **README.md** for anything user-facing: new variables, new behaviour, new failure modes worth a troubleshooting entry. Every variable in `defaults/main.yml` should appear in the variables tables. +- **CHANGELOG.md** for every change. Add entries under `## [Unreleased]` using the categories Added, Changed, Deprecated, Removed, Fixed and Security. Describe what changed and why it mattered, not just which file moved. +- **DEVELOPMENT.md** for anything that affects how contributors work on the collection. + +Two house style rules for all documentation in this repository: no em-dashes, and no emoji. + +AI-assisted contributions are accepted and reviewed on the same terms as any other change. + +Code comments should explain why, not what. If a task's name already says what it does, the comment should only exist to record the non-obvious reason it is written that way. + +## Versioning + +The collection follows [Semantic Versioning](https://semver.org/): + +- **Major**, for breaking changes: renamed or retyped variables, removed features, or changed behaviour that could affect an existing workflow. +- **Minor**, for backwards-compatible features: new variables with sensible defaults, new optional capabilities. +- **Patch**, for backwards-compatible fixes. + +New variables should default to off or empty, so that installing an upgrade never changes what an existing playbook run does. + +Do not bump the version in `galaxy.yml` in a pull request. Releases are cut separately, and the release workflow fails the build if a `v*` tag does not match the version in `galaxy.yml`. + +## Pull request checklist + +- [ ] `ansible-lint` passes with no new warnings +- [ ] `ansible-playbook playbooks/rotate.yml --syntax-check` passes +- [ ] `molecule test -s default` and `molecule test -s rollback` pass, or you have explained why they could not be run +- [ ] New behaviour is covered by a test that fails without the change +- [ ] The safety model is preserved, or any change to it is called out and justified +- [ ] README.md updated for user-facing changes +- [ ] CHANGELOG.md updated under `## [Unreleased]` +- [ ] No new dependencies without justification +- [ ] Backwards compatible, or the breaking change is documented +- [ ] No key material, private or public, committed + +## Reporting bugs + +Open an issue at [github.com/krameff/ssh_key_rotation/issues](https://github.com/krameff/ssh_key_rotation/issues) and include: + +- The target OS and version, and the output of `ansible --version` +- The command you ran, with key paths redacted +- The failing task name and the error, ideally with `-vvv` +- Whether the host was left in a working state + +If you have found a way to lock a host out, please treat it as a security issue rather than an ordinary bug, and email instead of opening a public issue. See [SECURITY.md](SECURITY.md). + +## Licence + +By contributing, you agree that your contributions will be licensed under the [MIT Licence](LICENSE) that covers this project. diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 1b1192e..0afddc4 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -2,23 +2,30 @@ Instructions for extending and testing the SSH Key Rotation collection. -## Project Structure +For the contribution workflow, the checks a change has to pass, and the pull request policy, see [CONTRIBUTING.md](CONTRIBUTING.md). This file covers the project layout, how to add code, and how releases are built. + +## Project structure ``` krameff-ssh_key_rotation/ ├── galaxy.yml # Collection metadata ├── README.md # Main documentation ├── QUICKSTART.md # Quick start guide +├── CONTRIBUTING.md # How to contribute, and what a change has to pass +├── SECURITY.md # Private disclosure route, and what counts as a security issue here ├── CHANGELOG.md # Version history ├── DEVELOPMENT.md # This file ├── requirements.yml # Ansible Galaxy dependencies -├── inventory.example.ini # Example inventory +├── inventory.example.ini # Example inventory +├── ansible.cfg # Sets roles_path so playbooks/rotate.yml finds roles/ without installing the collection ├── .ansible-lint # ansible-lint configuration (production profile) ├── .github/ +│ ├── pull_request_template.md +│ ├── ISSUE_TEMPLATE/ # Bug and feature forms; config.yml routes security reports privately │ └── workflows/ │ ├── ci.yml # Lint + syntax-check on push/PR -│ └── molecule.yml # Functional Molecule/Docker tests (default + rollback scenarios) on push/PR -├── ansible.cfg # Sets roles_path so playbooks/rotate.yml finds roles/ without installing the collection +│ ├── molecule.yml # Functional Molecule tests (default + rollback) on push/PR, plus a weekly run +│ └── release.yml # Builds and publishes to Galaxy on a v* tag ├── extensions/ │ └── molecule/ # Molecule scenarios (the location current Molecule expects) │ ├── default/ # Full rotation over real SSH, twice, to also prove idempotency @@ -33,128 +40,120 @@ krameff-ssh_key_rotation/ │ ├── handlers/main.yml # Shared "Reload sshd" handler │ ├── meta/main.yml # Role metadata (platforms, min Ansible version) │ └── tasks/ -│ ├── main.yml # Fails fast - this role has no default entry point, see below +│ ├── main.yml # Fails fast; this role has no default entry point, see below │ ├── validate.yml # Phase 0: local pre-flight validation │ ├── install.yml # Phase 1: install the new key, prepare sshd (connect via OLD key) │ ├── verify.yml # Phase 2: verify the new key, then remove the old key/legacy auth -│ └── manage_crypto_policy.yml # RHEL/Fedora crypto-policy logic, shared by validate.yml and install.yml via include_tasks +│ └── manage_crypto_policy.yml # RHEL/Fedora crypto-policy logic, shared by validate.yml and install.yml └── plugins/ ├── modules/ # Custom modules (future) └── filters/ # Custom filters (future) ``` -## Setup for Development +### Why there is no usable `tasks/main.yml` -### Prerequisites +Each stage of a rotation authenticates differently. Validate runs on the control node with no remote connection at all, install connects with the old key, and verify connects with the new one. There is no single set of connection variables that would work for all three, so the role deliberately has no default entry point and must be included with an explicit `tasks_from`. `tasks/main.yml` exists only to fail with a message saying so. -```bash -# Ansible 2.15+ -pip install 'ansible>=2.15' +That is also why `playbooks/rotate.yml` is three separate plays rather than one. -# ansible.posix collection -ansible-galaxy collection install ansible.posix +## Setting up -# Testing tools (optional but recommended) -pip install ansible-lint pytest-ansible molecule -``` +```bash +git clone https://github.com/krameff/ssh_key_rotation +cd ssh_key_rotation -### Clone and Setup +python -m venv .venv && source .venv/bin/activate -```bash -git clone -cd krameff-ssh_key_rotation +# Ansible, matching the minimum in meta/runtime.yml +pip install "ansible>=2.18" -# Install dependencies +# Testing tools +pip install ansible-lint "molecule>=25.0" "molecule-plugins[docker]" + +# Collection dependencies ansible-galaxy install -r requirements.yml -# Verify collection structure +# Sanity check the collection structure ansible-galaxy collection build . ``` -## Development Tasks +## Adding code -### Adding a New Playbook +### A new playbook -1. Create the playbook in `playbooks/`: - ```bash - touch playbooks/my_new_playbook.yml - ``` +1. Create it in `playbooks/`. +2. Document its usage in `README.md`. +3. Test it: `ansible-playbook -i inventory.ini playbooks/my_new_playbook.yml` -2. Update `README.md` with usage instructions +### A custom module -3. Test with: - ```bash - ansible-playbook -i inventory.ini playbooks/my_new_playbook.yml - ``` +1. Create it in `plugins/modules/`. +2. Document it in the module's `DOCUMENTATION` docstring. +3. Test it: `ansible localhost -m my_module -a "param=value"` -### Adding a Custom Module +### A custom filter -1. Create the module in `plugins/modules/`: - ```bash - touch plugins/modules/my_module.py - ``` +1. Create it in `plugins/filters/my_filters.py` and implement the filter function. +2. Test it: `ansible localhost -m debug -a "msg='{{ 'test' | my_filter }}'"` -2. Add module documentation in the docstring +## Testing -3. Test with: - ```bash - ansible localhost -m my_module -a "param=value" - ``` +### Lint and syntax -### Adding a Custom Filter +These are the two checks CI runs on every push and pull request. -1. Create the filter in `plugins/filters/my_filters.py` +```bash +# Lint the whole repo. rotate.yml is only the entry point, so linting it alone misses the role +ansible-lint -2. Implement the filter function +ansible-playbook playbooks/rotate.yml --syntax-check +``` -3. Test with: - ```bash - ansible localhost -m debug -a "msg='{{ 'test' | my_filter }}'" - ``` +`ansible-lint` runs on the `production` profile, configured in `.ansible-lint`. -## Testing - -### Syntax Check +### Dry run ```bash -ansible-playbook playbooks/rotate.yml --syntax-check +ansible-playbook -i inventory.ini playbooks/rotate.yml --check ``` -### Dry Run +### Molecule + +Two scenarios live under `extensions/molecule/`. They run against live containers (Ubuntu 22.04, Rocky Linux 9, Rocky Linux 10) with sshd installed and running. ```bash -ansible-playbook -i inventory.ini playbooks/rotate.yml --check +# Full rotation via playbooks/rotate.yml, run twice to also prove idempotency +molecule test -s default + +# Runs the role's stages directly, breaking the second rotation mid-verify to prove the +# block/rescue in roles/ssh_key_rotation/tasks/verify.yml actually restores access +molecule test -s rollback ``` -### Lint with ansible-lint +Both scenarios generate their own ephemeral SSH keypairs and inventory into Molecule's per-run directory, so nothing needs to exist in the repository for them to run. + +While iterating, avoid the full create-and-destroy cycle each time: ```bash -# rotate.yml is just the entry point; lint the whole repo to cover the role too -ansible-lint +molecule converge -s default # run against live containers +molecule verify -s default # run just the assertions +molecule destroy -s default # clean up ``` -### Integration Testing with Molecule +The scenarios track the `geerlingguy` CI images by `:latest`, so the base images move underneath the suite between commits. `molecule.yml` runs weekly for that reason, so drift surfaces as its own red build rather than as a mystery failure on an unrelated pull request. -Two scenarios live under `extensions/molecule/` and run the real `playbooks/rotate.yml` (or, for -the rollback scenario, the role's stages directly) against live Docker containers -(Ubuntu 22.04, Rocky Linux 9, Rocky Linux 10) with sshd installed and running: +### Two ways a test here can silently never fail -```bash -pip install "molecule>=25.0" "molecule-plugins[docker]" +Both of these have happened in this repository and are worth checking for in any new assertion: -# Full rotation + idempotency check -molecule test -s default +- **The container connection ignores SSH keys.** A task run over Molecule's `docker` or `podman` connection proves the host is reachable, not that a key authenticates. To test authentication, drive a real `ssh` client. +- **Key material is not a safe regex.** Base64 key data contains regex metacharacters, so a pattern built from a key can match when the key is still present. Match it literally. -# Deliberately breaks the second rotation mid-verify to prove the block/rescue -# in roles/ssh_key_rotation/tasks/verify.yml actually restores access -molecule test -s rollback -``` +Before relying on a new assertion, break the thing it checks and confirm it goes red. -Both scenarios generate their own ephemeral SSH keypairs and inventory into Molecule's -per-run ephemeral directory, so nothing needs to exist in the repository for them to run. +### Manual testing -For local manual testing, generate your own throwaway keypairs outside the repository and -point the playbook at them: +Generate throwaway keys outside the repository and point the playbook at them: ```bash ssh-keygen -t rsa -b 4096 -N "" -f /tmp/rotation-old @@ -167,45 +166,19 @@ ansible-playbook -i inventory.ini playbooks/rotate.yml \ -e new_public_key_file=/tmp/rotation-new.pub ``` -Keep generated keys out of the working tree. `.gitignore` covers the common patterns, but -a private key that lands in a commit has to be treated as compromised regardless. - -## Building and Publishing - -### Build a Distribution Tarball - -```bash -ansible-galaxy collection build . -# Output: ./krameff-ssh_key_rotation-1.0.0.tar.gz -``` - -### Publish to Ansible Galaxy - -```bash -# Requires API token -ansible-galaxy collection publish ./krameff-ssh_key_rotation-1.0.0.tar.gz \ - --api-key -``` +Use a disposable VM you can snapshot and roll back. -## Updating Documentation +Keep generated keys out of the working tree. `.gitignore` covers the common patterns and `galaxy.yml`'s `build_ignore` excludes `test_*` as a second line of defence, but a private key that lands in a commit has to be treated as compromised regardless. -### Update README +## Documentation -1. Keep the overview concise but comprehensive -2. Include all required and optional variables -3. Add examples for common use cases -4. Update troubleshooting if adding new features +**README.md** should stay the reference for anything user-facing. Keep the overview concise, list every required and optional variable, include examples for common cases, and add a troubleshooting entry for any new failure mode. -### Update CHANGELOG +**CHANGELOG.md** gets an entry for every change, under `## [Unreleased]`, using the categories Added, Changed, Deprecated, Removed, Fixed and Security. Link to issues or pull requests where relevant. On release, entries move from `Unreleased` into a versioned section. -1. Add entries under an "Unreleased" section -2. Use categories: Added, Changed, Deprecated, Removed, Fixed, Security -3. Link to GitHub issues/PRs when applicable -4. Release notes move from "Unreleased" to a versioned section +**House style**: no em-dashes and no emoji, in any document in this repository. -### Add Code Comments - -Keep comments minimal and focused on WHY, not WHAT: +**Code comments** should explain why, not what: ```yaml # Good: explains the non-obvious reason for a choice @@ -220,63 +193,48 @@ Keep comments minimal and focused on WHY, not WHAT: ## Versioning -This collection follows [Semantic Versioning](https://semver.org/): +The collection follows [Semantic Versioning](https://semver.org/): -- **MAJOR** (1.0.0 → 2.0.0): Breaking changes - - Changes to variable names or types - - Removal of playbooks or features - - Changed behavior that could affect existing workflows - -- **MINOR** (1.0.0 → 1.1.0): New features, backwards compatible - - New playbooks or roles - - New variables with sensible defaults - - New optional capabilities - -- **PATCH** (1.0.0 → 1.0.1): Bug fixes, backwards compatible - - Configuration fixes - - Documentation fixes - - Small improvements that don't change behavior +- **Major** (1.0.0 to 2.0.0), for breaking changes: renamed or retyped variables, removed playbooks or features, or changed behaviour that could affect an existing workflow. +- **Minor** (1.0.0 to 1.1.0), for backwards-compatible features: new playbooks or roles, new variables with sensible defaults, new optional capabilities. +- **Patch** (1.0.0 to 1.0.1), for backwards-compatible fixes: configuration fixes, documentation fixes, small improvements that do not change behaviour. -## Contributing +New variables should default to off or empty, so that upgrading never changes what an existing playbook run does. -### Before Submitting +## Building and releasing -1. Run all tests: - ```bash - ansible-lint - ansible-playbook playbooks/rotate.yml --syntax-check - ``` +### Build a tarball -2. Test against multiple OS families (Debian, RHEL) +```bash +ansible-galaxy collection build . +# Output: ./krameff-ssh_key_rotation-1.0.0.tar.gz +``` -3. Update documentation: - - README.md with new variables/options - - CHANGELOG.md with changes - - Code comments for non-obvious logic +`galaxy.yml`'s `build_ignore` keeps the Molecule scenarios, CI configuration, editor and agent directories, and any local inventory or key material out of the tarball. -4. Verify backwards compatibility +### Release -### Pull Request Checklist +Releases are cut by tagging, not by hand. Pushing a `v*` tag triggers `.github/workflows/release.yml`, which builds the collection and publishes it to Galaxy using the `GALAXY_API_KEY` repository secret. The workflow fails the build if the tag does not match the version in `galaxy.yml`. -- [ ] Code follows Ansible best practices -- [ ] All tests pass -- [ ] Documentation updated -- [ ] CHANGELOG.md updated -- [ ] No new dependencies without justification -- [ ] Backwards compatible (or documented breaking change) +So the order is: bump `version` in `galaxy.yml`, move the `Unreleased` changelog entries into a versioned section, merge, then tag. -## Future Enhancements +To publish by hand if you need to: -See [CHANGELOG.md](CHANGELOG.md) for planned features: +```bash +ansible-galaxy collection publish ./krameff-ssh_key_rotation-1.0.0.tar.gz \ + --api-key +``` + +## Future enhancements -- Basic key type/strength/pairing validation now runs in `roles/ssh_key_rotation/tasks/validate.yml`; richer checks (permissions, passphrase detection) are still future work -- Per-OS customization -- Key integrity validation module -- Automatic rollback in the verify stage's cleanup now runs via `block`/`rescue`, restoring from the backups taken moments earlier on the same connection; a standalone break-glass rollback playbook (for recovering a host after the connection is already lost) is still future work +- Richer Phase 0 key validation. Type, strength and pairing checks are in `roles/ssh_key_rotation/tasks/validate.yml` already; permissions and passphrase detection are not. +- A standalone break-glass rollback playbook. The verify stage's `block`/`rescue` restores from backups on the same still-open connection, but there is nothing to recover a host once that connection is already lost. +- Per-OS customisation +- A key integrity validation module - Multi-key rotation support - Async key rotation for large fleets -## Troubleshooting Development +## Troubleshooting development ### Collection import errors @@ -290,19 +248,15 @@ ansible-galaxy collection build . --check ### Module not found +Make sure the module is in `plugins/modules/`, then clear the cache and retry: + ```bash -# Ensure module is in plugins/modules/ -# Restart Ansible or clear cache rm -rf ~/.ansible/plugins/modules/ ``` ### Filter not loading -```bash -# Verify filter file is in plugins/filters/ -# Check function name matches filter invocation -# Reload Ansible cache -``` +Check the filter file is in `plugins/filters/`, that the function name matches the filter invocation, and reload the Ansible cache. ## Resources diff --git a/README.md b/README.md index 465f1fb..832faf5 100644 --- a/README.md +++ b/README.md @@ -1,51 +1,88 @@ +

+ + +

+ # SSH Key Rotation Collection -A production-safe Ansible collection for rotating SSH keys across your infrastructure without breaking existing connections. +An Ansible collection for rotating SSH keys across your infrastructure without breaking existing connections. + +> **Warning** +> +> As with anything you download, test this thoroughly against a disposable host before running it anywhere that matters. A VM snapshot you can roll back is ideal. +> +> The collection is built with multiple safety gates (see [Safety model](#safety-model)), but no automated tool can guarantee it fits every environment, and SSH lockouts are painful to recover from. + +## Contents + +- [How it works](#how-it-works) +- [Features](#features) +- [Requirements](#requirements) +- [Tested platforms](#tested-platforms) +- [Installation](#installation) +- [Usage](#usage) +- [Variables](#variables) +- [Key validation](#key-validation) +- [PQC algorithm negotiation](#pqc-algorithm-negotiation) +- [Safety model](#safety-model) +- [What each phase does](#what-each-phase-does) +- [Role reference](#role-reference) +- [Troubleshooting](#troubleshooting) +- [Development](#development) + +## How it works + +Rotating SSH keys is easy to get wrong in a way that locks you out of a server. This collection splits the job into three phases so that a mistake at any point fails safely instead of leaving a host unreachable. -> **Warning:** as with anything you download, test this thoroughly against a disposable or non-critical host (a VM snapshot you can roll back is ideal) before running it anywhere that matters. This collection is built with multiple safety gates (see [Safety model](#safety-model)), but no automated tool can guarantee it fits every environment, and SSH lockouts can be genuinely painful to recover from. +**Phase 0: validate.** Runs entirely on the control node, before touching a single host. Checks that the required variables are set, that the new key is a recognised and strong type, and that the new private key really matches the public key file. -## Overview +**Phase 1: install.** Connects with the old key, installs the new key alongside it, and makes sure public key authentication is enabled. -Rotating SSH keys is one of those jobs that's easy to get wrong in a way that locks you out of a server. This collection handles it as a three-phase process, specifically so that a mistake at any point fails safely instead of leaving a host unreachable: +**Phase 2: verify.** Reconnects using the *new* key to prove it works. Only then does it remove the old key and disable legacy auth methods. -0. **Phase 0** - Validate everything locally, before touching a single host: required variables are set, the new key is a recognized and strong type, and the new private key actually matches the public key file. -1. **Phase 1** - Connect with the old key, install the new key alongside it, and make sure public key authentication is enabled. -2. **Phase 2** - Reconnect using the *new* key to prove it actually works, then (and only then) remove the old key and disable legacy auth methods. +The old key is never touched until the new one has proven it can authenticate. If Phase 2 cannot connect with the new key, the playbook stops there and makes no further edits to `authorized_keys` or `sshd_config` on that host. -The old key is never touched until the new one has proven it can authenticate. If Phase 2 can't connect with the new key, the playbook stops right there - it won't go on to edit `authorized_keys` or `sshd_config` any further on that host. Every `sshd_config` change is checked with `sshd -t` before it's written, and both `sshd_config` and `authorized_keys` are backed up first, so a bad change can always be reverted by hand afterwards. +Every `sshd_config` change is checked with `sshd -t` before it is written, and both `sshd_config` and `authorized_keys` are backed up first, so a bad change can always be reverted by hand. ## Features -- **Zero-downtime** - uses an `sshd` reload rather than a restart, so existing SSH sessions survive the configuration change -- **Key recognition checks** - rejects deprecated or weak key types and undersized RSA keys before rotating anything (see [Key Validation](#key-validation-pqc-readiness)) -- **Hard safety gate** - the new key must authenticate before the old one is removed -- **Cross-OS support** - handles the Debian/Ubuntu vs RHEL/CentOS `sshd` service-name difference for you -- **Drop-in and `Match` block awareness** - warns if `/etc/ssh/sshd_config.d/` files or `Match` blocks might silently override what this playbook is setting -- **Flexible auth policy** - optionally make the new key exclusive, and optionally disable password/keyboard-interactive auth -- **Validated, backed-up config edits** - every `sshd` change is checked with `sshd -t` and backed up before it's written -- **PQC algorithm negotiation (opt-in)** - can also enable post-quantum/hybrid key exchange and signature algorithms across `sshd_config`, RHEL/Fedora crypto-policy, and the control node's own ssh client (see [PQC Algorithm Negotiation](#pqc-algorithm-negotiation)) +**Zero downtime.** Configuration is applied with an `sshd` reload rather than a restart, so existing SSH sessions survive the change. + +**Key recognition checks.** Deprecated key types, weak key types and undersized RSA keys are rejected before anything is rotated. See [Key validation](#key-validation). + +**A hard safety gate.** The new key must authenticate before the old one is removed. + +**Cross-OS support.** The Debian/Ubuntu vs RHEL/CentOS `sshd` service-name difference is handled for you. + +**Drop-in and `Match` block awareness.** Warns if files in `/etc/ssh/sshd_config.d/` or `Match` blocks might silently override what the playbook is setting. + +**Flexible auth policy.** Optionally make the new key exclusive, and optionally disable password and keyboard-interactive auth. + +**Validated, backed-up config edits.** Every `sshd` change is checked with `sshd -t` and backed up before it is written. + +**Post-quantum negotiation, opt-in.** Can enable post-quantum and hybrid key exchange and signature algorithms across `sshd_config`, RHEL/Fedora crypto-policy, and the control node's own ssh client. See [PQC algorithm negotiation](#pqc-algorithm-negotiation). ## Requirements -- **Ansible** ≥ 2.18 -- **ansible.posix** ≥ 1.3.0 +- Ansible 2.18 or later +- `ansible.posix` 1.3.0 or later - ```bash - ansible-galaxy collection install ansible.posix - ``` +```bash +ansible-galaxy collection install ansible.posix +``` -## Tested Platforms +## Tested platforms -This collection targets Debian/Ubuntu and RHEL/Fedora-family targets generally, but these are the specific OS versions it's actually been run against, end to end, on real hosts: +The collection targets Debian/Ubuntu and RHEL/Fedora-family hosts generally. These are the specific versions it has been run against end to end, on real hosts: | OS | Notes | |----|-------| -| Ubuntu 22.04 LTS (Jammy Jellyfish) | Watch for `sshd_config.d/` drop-ins (e.g. cloud-init's `50-cloud-init.conf`) that can override settings this playbook writes further down `sshd_config` - see [Drop-in sshd config files found](#drop-in-sshd-config-files-found) | -| AlmaLinux 9.8 (Olive Jaguar), `FIPS` crypto-policy | `FIPS` alone has no PQC key exchange; use `ssh_key_rotation_crypto_policy_add_modules: [PQ]` to add it (see [Combining a base policy with a subpolicy module](#combining-a-base-policy-with-a-subpolicy-module-basemodule)) | -| AlmaLinux 10.2 (Lavender Lion), `FIPS` crypto-policy | `FIPS` already includes PQC key exchange by default here, no extra module needed | -| openSUSE Leap 15.6 | Ships Python 3.6 by default, which is too old for Ansible ≥ 2.15's target-side requirements; set `ansible_python_interpreter` to a Python 3.7+ install (e.g. `python39` via `zypper`) | +| Ubuntu 22.04 LTS (Jammy Jellyfish) | Watch for `sshd_config.d/` drop-ins such as cloud-init's `50-cloud-init.conf`, which can override settings written further down `sshd_config`. See [Drop-in sshd config files found](#drop-in-sshd-config-files-found). | +| AlmaLinux 9.8 (Olive Jaguar), `FIPS` crypto-policy | `FIPS` on its own has no PQC key exchange. Use `ssh_key_rotation_crypto_policy_add_modules: [PQ]` to add it. See [Combining a base policy with a subpolicy module](#combining-a-base-policy-with-a-subpolicy-module). | +| AlmaLinux 10.2 (Lavender Lion), `FIPS` crypto-policy | `FIPS` already includes PQC key exchange here. No extra module needed. | +| openSUSE Leap 15.6 | Ships Python 3.6 by default, which is too old for Ansible 2.15+ on the target side. Set `ansible_python_interpreter` to a Python 3.7+ install, for example `python39` via `zypper`. | -Other versions in the same OS families are likely to work too, since nothing here relies on version-specific behavior beyond what's called out above, but they haven't been explicitly verified. +Other versions in the same families are likely to work, since nothing here relies on version-specific behaviour beyond what is called out above. They have not been explicitly verified. ## Installation @@ -71,37 +108,16 @@ ansible-playbook krameff.ssh_key_rotation.rotate \ --ask-become-pass ``` -### Required variables - -| Variable | Type | Description | -|----------|------|-------------| -| `old_private_key` | string | Path to the old SSH private key (local) | -| `new_private_key` | string | Path to the new SSH private key (local) | -| `old_public_key_file` | string | Path to the old SSH public key (local) | -| `new_public_key_file` | string | Path to the new SSH public key (local) | +The playbook runs against the `rotate` host group, which you define in your inventory: -### Optional variables - -| Variable | Type | Default | Description | -|----------|------|---------|-------------| -| `ansible_user` | string | $USER | Remote user to rotate keys for | -| `ssh_key_rotation_disable_password_auth` | bool | true | Disable password authentication after rotation | -| `ssh_key_rotation_disable_kbd_interactive` | bool | true | Disable keyboard-interactive auth after rotation | -| `ssh_key_rotation_make_exclusive` | bool | false | Leave ONLY the new key in authorized_keys | -| `ssh_key_rotation_accepted_key_types` | list | `[ED25519, ED25519-SK, ECDSA, ECDSA-SK, RSA]` | Key types allowed for `new_public_key_file` | -| `ssh_key_rotation_pqc_key_types` | list | `[]` | Forward-compatible allowlist slot for post-quantum signature key types (see [Key Validation](#key-validation-pqc-readiness)) | -| `ssh_key_rotation_reject_key_types` | list | `[DSA]` | Key types that always fail validation, regardless of `ssh_key_rotation_accepted_key_types` | -| `ssh_key_rotation_min_rsa_bits` | int | `3072` | Minimum RSA key size accepted | -| `ssh_key_rotation_pqc_kex_algorithms` | list | `[]` | PQC/hybrid key-exchange algorithm names (see [PQC Algorithm Negotiation](#pqc-algorithm-negotiation)) | -| `ssh_key_rotation_pqc_pubkey_algorithms` | list | `[]` | PQC/hybrid signature algorithm names for `PubkeyAcceptedAlgorithms`/`HostKeyAlgorithms` | -| `ssh_key_rotation_pqc_ca_signature_algorithms` | list | `[]` | PQC/hybrid algorithm names for `CASignatureAlgorithms` (SSH CA setups only) | -| `ssh_key_rotation_manage_crypto_policy` | bool | `false` | Opt-in: manage RHEL/Fedora system-wide crypto-policy on the control node and targets | -| `ssh_key_rotation_crypto_policy_setting` | string | `DEFAULT:PQ` | Value passed to `update-crypto-policies --set` when `ssh_key_rotation_manage_crypto_policy` is true | -| `ssh_key_rotation_crypto_policy_add_modules` | list | `[]` | Adds module(s) (e.g. `PQ`) onto whatever policy a host already has, instead of replacing it (see [Combining a base policy with a subpolicy module](#combining-a-base-policy-with-a-subpolicy-module-basemodule)) | - -### Advanced examples +```ini +[rotate] +prod-web-01 ansible_host=10.0.1.10 +prod-web-02 ansible_host=10.0.1.11 +prod-db-01 ansible_host=10.0.2.10 +``` -#### Disable password auth but keep keyboard-interactive +### Disable password auth but keep keyboard-interactive ```bash ansible-playbook krameff.ssh_key_rotation.rotate \ @@ -114,7 +130,7 @@ ansible-playbook krameff.ssh_key_rotation.rotate \ -e ssh_key_rotation_disable_kbd_interactive=false ``` -#### Keep multiple keys (don't make exclusive) +### Keep the old key in place as well as the new one ```bash ansible-playbook krameff.ssh_key_rotation.rotate \ @@ -127,38 +143,87 @@ ansible-playbook krameff.ssh_key_rotation.rotate \ -e ssh_key_rotation_disable_password_auth=true ``` -## Key Validation (PQC readiness) +## Variables + +### Required + +| Variable | Type | Description | +|----------|------|-------------| +| `old_private_key` | string | Path to the old SSH private key, on the control node | +| `new_private_key` | string | Path to the new SSH private key, on the control node | +| `old_public_key_file` | string | Path to the old SSH public key, on the control node | +| `new_public_key_file` | string | Path to the new SSH public key, on the control node | + +### Optional + +| Variable | Type | Default | Description | +|----------|------|---------|-------------| +| `ansible_user` | string | `$USER` | Remote user to rotate keys for | +| `ssh_key_rotation_disable_password_auth` | bool | `true` | Disable password authentication after rotation | +| `ssh_key_rotation_disable_kbd_interactive` | bool | `true` | Disable keyboard-interactive auth after rotation | +| `ssh_key_rotation_make_exclusive` | bool | `false` | Leave only the new key in `authorized_keys` | +| `ssh_key_rotation_accepted_key_types` | list | `[ED25519, ED25519-SK, ECDSA, ECDSA-SK, RSA]` | Key types allowed for `new_public_key_file` | +| `ssh_key_rotation_reject_key_types` | list | `[DSA]` | Key types that always fail validation, whatever the accepted list says | +| `ssh_key_rotation_min_rsa_bits` | int | `3072` | Minimum RSA key size accepted | +| `ssh_key_rotation_pqc_key_types` | list | `[]` | Forward-compatible allowlist for post-quantum signature key types. See [Key validation](#key-validation). | + +### Optional, post-quantum + +All of these are empty or false by default, so none of them change existing behaviour unless you ask for them. See [PQC algorithm negotiation](#pqc-algorithm-negotiation). + +| Variable | Type | Default | Description | +|----------|------|---------|-------------| +| `ssh_key_rotation_pqc_kex_algorithms` | list | `[]` | PQC or hybrid key-exchange algorithm names | +| `ssh_key_rotation_pqc_pubkey_algorithms` | list | `[]` | PQC or hybrid signature algorithm names for `PubkeyAcceptedAlgorithms` and `HostKeyAlgorithms` | +| `ssh_key_rotation_pqc_ca_signature_algorithms` | list | `[]` | PQC or hybrid algorithm names for `CASignatureAlgorithms`, for SSH CA setups only | +| `ssh_key_rotation_manage_crypto_policy` | bool | `false` | Manage the RHEL/Fedora system-wide crypto-policy on the control node and targets | +| `ssh_key_rotation_crypto_policy_setting` | string | `DEFAULT:PQ` | Value passed to `update-crypto-policies --set` | +| `ssh_key_rotation_crypto_policy_add_modules` | list | `[]` | Adds modules such as `PQ` onto whatever policy a host already has, instead of replacing it | -Before any host is touched, Phase 0 runs entirely on the control node and checks: +## Key validation -1. All four required variables are set (fails fast with a clear message instead of a deep "undefined variable" error) -2. `new_public_key_file`'s type (via `ssh-keygen -l`) is not in `ssh_key_rotation_reject_key_types` (e.g. `ssh-dss`/DSA) -3. Its type is in `ssh_key_rotation_accepted_key_types` **or** `ssh_key_rotation_pqc_key_types` -4. If it's an RSA key, it meets `ssh_key_rotation_min_rsa_bits` -5. `new_private_key` and `new_public_key_file` are actually a matching pair (fingerprints must agree) - this catches the common mistake of pointing at mismatched key files +Before any host is touched, Phase 0 runs on the control node and checks that: -**On post-quantum cryptography:** mainline OpenSSH doesn't yet ship a post-quantum *signature* algorithm for `authorized_keys`/host keys, only post-quantum *key exchange* for the transport layer (`mlkem768x25519-sha256`, `sntrup761x25519-sha512`); see [openssh.org/pq.html](https://www.openssh.org/pq.html). So there isn't a standard "PQC key" to check for yet. `ssh_key_rotation_pqc_key_types` exists as a forward-compatible allowlist: once OpenSSH (or an [OQS-OpenSSH](https://github.com/open-quantum-safe/openssh) build) reports a PQC signature type such as `MLDSA65` or `FALCON1024` from `ssh-keygen -l`, just add that name to `ssh_key_rotation_pqc_key_types` (via `-e` or by editing `roles/ssh_key_rotation/defaults/main.yml`) and it'll be accepted, no other playbook change required. +1. All four required variables are set. This fails fast with a clear message rather than a deep "undefined variable" error. +2. The type of `new_public_key_file`, read via `ssh-keygen -l`, is not in `ssh_key_rotation_reject_key_types`, for example `ssh-dss`/DSA. +3. That type is in `ssh_key_rotation_accepted_key_types` or `ssh_key_rotation_pqc_key_types`. +4. If it is an RSA key, it meets `ssh_key_rotation_min_rsa_bits`. +5. `new_private_key` and `new_public_key_file` are genuinely a matching pair, by comparing fingerprints. This catches the common mistake of pointing at mismatched key files. -Phase 1 also does a hard check after reloading `sshd`: it runs `sshd -T` on the target, extracts the effective `PubkeyAcceptedAlgorithms` list, and **fails the host** if none of the new key type's real signature algorithms (e.g. `ssh-ed25519` for an ED25519 key) are in it. This matters because a system-wide crypto-policy can silently narrow what the *type check* in Phase 0 would otherwise accept - for example, RHEL/Fedora's `FIPS` policy drops `ssh-ed25519` entirely and only allows `ecdsa-sha2-*`/`rsa-sha2-*`. Without this check, Phase 1 would report success, and Phase 2 would go on to remove the old key anyway, leaving the host with no key it can actually authenticate with. The check runs, and can fail the host, *before* Phase 2 does anything destructive. +### A note on post-quantum keys -## PQC Algorithm Negotiation +Mainline OpenSSH does not yet ship a post-quantum *signature* algorithm for `authorized_keys` or host keys. It only ships post-quantum *key exchange* for the transport layer, namely `mlkem768x25519-sha256` and `sntrup761x25519-sha512`. See [openssh.org/pq.html](https://www.openssh.org/pq.html). -`ssh_key_rotation_pqc_key_types` above only affects local key *type* recognition in Phase 0 - it never changes what `sshd`/`ssh` actually negotiate on the wire. For a PQC or hybrid key to actually work end to end, both sides of the connection need matching algorithm lists across several independent layers, and this collection can manage all of them. Everything here is opt-in, and empty/false by default, so none of it changes existing behaviour unless you ask for it: +So there is no standard "PQC key" to check for yet. `ssh_key_rotation_pqc_key_types` exists as a forward-compatible allowlist. Once OpenSSH, or an [OQS-OpenSSH](https://github.com/open-quantum-safe/openssh) build, reports a PQC signature type such as `MLDSA65` or `FALCON1024` from `ssh-keygen -l`, add that name to `ssh_key_rotation_pqc_key_types` and it will be accepted. No other change is needed. -| Variable | Affects | -|----------|---------| -| `ssh_key_rotation_pqc_kex_algorithms` | `KexAlgorithms` in the target's `sshd_config`, and `-o KexAlgorithms` for this playbook's own ssh connections | -| `ssh_key_rotation_pqc_pubkey_algorithms` | `PubkeyAcceptedAlgorithms`/`HostKeyAlgorithms` in the target's `sshd_config`, and `-o PubkeyAcceptedAlgorithms` for this playbook's own ssh connections | -| `ssh_key_rotation_pqc_ca_signature_algorithms` | `CASignatureAlgorithms` in the target's `sshd_config` (SSH CA setups only) | -| `ssh_key_rotation_manage_crypto_policy` | Also manage RHEL/Fedora system-wide crypto-policy, on the control node and on targets | -| `ssh_key_rotation_crypto_policy_setting` | Value passed to `update-crypto-policies --set`, optionally combining a base policy with one or more subpolicy modules via `BASE:MODULE` syntax (e.g. `DEFAULT:PQ`, `FIPS:PQ`) | -| `ssh_key_rotation_crypto_policy_add_modules` | Preferred over `ssh_key_rotation_crypto_policy_setting`: adds module(s) onto a host's current policy instead of replacing it | +### The Phase 1 algorithm check -Every `sshd_config` directive is written using OpenSSH's `+algorithm` syntax, appending to the compiled-in defaults rather than replacing the list outright, so clients that don't speak PQC yet can still fall back to a classical algorithm. +After reloading `sshd`, Phase 1 runs `sshd -T` on the target, extracts the effective `PubkeyAcceptedAlgorithms` list, and fails the host if none of the new key type's real signature algorithms are in it. For an ED25519 key, that means looking for `ssh-ed25519`. -### Where each piece lives +This matters because a system-wide crypto-policy can quietly narrow what the Phase 0 type check would otherwise accept. RHEL and Fedora's `FIPS` policy, for instance, drops `ssh-ed25519` entirely and allows only `ecdsa-sha2-*` and `rsa-sha2-*`. -Split by which stage actually runs each step, and on which machine (see [Role Reference](#role-reference) for how these map to `tasks/*.yml`): +Without this check, Phase 1 would report success and Phase 2 would go on to remove the old key, leaving the host with no key it can actually authenticate with. The check runs, and can fail the host, before Phase 2 does anything destructive. + +## PQC algorithm negotiation + +`ssh_key_rotation_pqc_key_types` only affects local key *type* recognition in Phase 0. It never changes what `sshd` and `ssh` actually negotiate on the wire. + +For a PQC or hybrid key to work end to end, both sides of the connection need matching algorithm lists across several independent layers. This collection can manage all of them: + +| Variable | What it affects | +|----------|-----------------| +| `ssh_key_rotation_pqc_kex_algorithms` | `KexAlgorithms` in the target's `sshd_config`, and `-o KexAlgorithms` for this playbook's own connections | +| `ssh_key_rotation_pqc_pubkey_algorithms` | `PubkeyAcceptedAlgorithms` and `HostKeyAlgorithms` in the target's `sshd_config`, and `-o PubkeyAcceptedAlgorithms` for this playbook's own connections | +| `ssh_key_rotation_pqc_ca_signature_algorithms` | `CASignatureAlgorithms` in the target's `sshd_config`, for SSH CA setups only | +| `ssh_key_rotation_manage_crypto_policy` | Whether the RHEL/Fedora system-wide crypto-policy is also managed, on the control node and on targets | +| `ssh_key_rotation_crypto_policy_setting` | The value passed to `update-crypto-policies --set`, optionally combining a base policy with subpolicy modules using `BASE:MODULE` syntax, such as `DEFAULT:PQ` or `FIPS:PQ` | +| `ssh_key_rotation_crypto_policy_add_modules` | Preferred over the setting above: adds modules onto a host's current policy instead of replacing it | + +Every `sshd_config` directive is written with OpenSSH's `+algorithm` syntax, appending to the compiled-in defaults rather than replacing the list outright, so clients that do not speak PQC yet can still fall back to a classical algorithm. + +### Where each piece runs + +The diagram below splits the work by stage and by machine. See [Role reference](#role-reference) for how these map to `tasks/*.yml`. ```mermaid flowchart TD @@ -203,21 +268,25 @@ flowchart TD EffectiveCheck --> Reconnect ``` -It starts on the control node, in Phase 0: `ssh -Q kex` and `ssh -Q key-sig` confirm your own ssh binary can actually offer the algorithms you're asking for, before any host is touched. If `ssh_key_rotation_manage_crypto_policy` is set and this happens to be a RHEL/Fedora control node, it also runs `update-crypto-policies --set` there, so the machine's ssh *client* backend permits PQC algorithms system-wide - this is detected by checking whether the `update-crypto-policies` tool exists, not by an OS-family fact. +**On the control node, in Phase 0.** `ssh -Q kex` and `ssh -Q key-sig` confirm your own ssh binary can offer the algorithms you are asking for, before any host is touched. If `ssh_key_rotation_manage_crypto_policy` is set and this is a RHEL or Fedora control node, `update-crypto-policies --set` runs there too, so the machine's ssh *client* backend permits PQC algorithms system-wide. This is detected by checking whether the `update-crypto-policies` tool exists, not by an OS-family fact. + +**On the connection itself.** The algorithms need to travel with the Ansible connection. Rather than editing any file on the control node, Phases 1 and 2 compute an `ansible_ssh_extra_args` value that passes `-o KexAlgorithms=+...` and `-o PubkeyAcceptedAlgorithms=+...` for this playbook's connections only. + +**On the target, in Phase 1.** `KexAlgorithms`, `PubkeyAcceptedAlgorithms`, `HostKeyAlgorithms` and `CASignatureAlgorithms` are appended to `sshd_config`, using the same `lineinfile` plus `sshd -t` plus backup pattern used everywhere else. -From there, the algorithms need to travel with the Ansible connection itself. Rather than editing any file on the control node, Phase 1 and Phase 2 compute an `ansible_ssh_extra_args` value that passes `-o KexAlgorithms=+...` and `-o PubkeyAcceptedAlgorithms=+...` for this playbook's own connections only. +If `ssh_key_rotation_manage_crypto_policy` is set and the target has the tooling, the crypto-policy step runs there too. Because `update-crypto-policies --set` validates only its own module syntax and not the resulting merged `sshd_config`, there is an explicit `sshd -t` re-check afterwards, before the reload handler is allowed to fire. The drop-in warning also calls out `50-redhat.conf` by name, since that file is the generated crypto-policy backend include and is not meant to be hand-edited. -On the target side, Phase 1 appends `KexAlgorithms`, `PubkeyAcceptedAlgorithms`, `HostKeyAlgorithms`, and `CASignatureAlgorithms` to `sshd_config`, using the same `lineinfile` plus `sshd -t` validation plus backup pattern used everywhere else in this playbook. If `ssh_key_rotation_manage_crypto_policy` is also set and the target has the tooling for it, the same RHEL/Fedora crypto-policy step runs there too - and because `update-crypto-policies --set` only validates its own module syntax (not the resulting merged `sshd_config`), there's an explicit `sshd -t` re-check afterwards, before the reload handler is allowed to fire. The existing drop-in warning also now calls out `50-redhat.conf` by name, since that file is the generated crypto-policy backend include and isn't meant to be hand-edited. +Once `sshd` has reloaded, the `sshd -T` check flags any requested algorithm that still is not showing up in the effective config, so you find out before Phase 2 tries and fails to reconnect. -#### Combining a base policy with a subpolicy module (`BASE:MODULE`) +### Combining a base policy with a subpolicy module -`update-crypto-policies --set` accepts a base policy name on its own (`DEFAULT`, `FIPS`, `LEGACY`, ...) or a base policy combined with one or more subpolicy *modules*, written as `BASE:MODULE` (e.g. `FIPS:PQ`, or `FIPS:PQ:NO-SHA1` to stack more than one). Each module is a `MODULE.pmod` file, either shipped by the OS under `/usr/share/crypto-policies/policies/modules/` or dropped in locally under `/etc/crypto-policies/policies/modules/`, and it's *added on top of* the base policy rather than replacing it - so `FIPS:PQ` stays FIPS-compliant everywhere else and only adds what `PQ.pmod` grants. +`update-crypto-policies --set` accepts either a base policy name on its own (`DEFAULT`, `FIPS`, `LEGACY` and so on) or a base policy combined with one or more subpolicy *modules*, written as `BASE:MODULE`. For example `FIPS:PQ`, or `FIPS:PQ:NO-SHA1` to stack more than one. -This matters specifically for PQC: on AlmaLinux/RHEL 9, the `FIPS` policy alone does not include any post-quantum key-exchange groups, but the OS still ships a built-in `PQ.pmod` module that adds `mlkem768x25519-sha256` (and other ML-KEM groups) when combined as `FIPS:PQ` - confirmed against a real AlmaLinux 9.8 host, where `sshd -T` only showed `mlkem768x25519-sha256` in the effective `KexAlgorithms` *after* switching from `FIPS` to `FIPS:PQ`. AlmaLinux/RHEL 10's `FIPS` policy ships with PQC key exchange already included, so this combination isn't needed there. +Each module is a `MODULE.pmod` file, either shipped by the OS under `/usr/share/crypto-policies/policies/modules/` or dropped in locally under `/etc/crypto-policies/policies/modules/`. A module is added on top of the base policy rather than replacing it, so `FIPS:PQ` stays FIPS-compliant everywhere else and only adds what `PQ.pmod` grants. -Before ever calling `update-crypto-policies --set`, this playbook lists whatever `*.pmod` files actually exist under both module directories on that host (control node in Phase 0, target in Phase 1) and, if `ssh_key_rotation_crypto_policy_setting` names a module, **fails before making any change** if that module isn't present - rather than letting `update-crypto-policies` silently ignore an unknown module name or fail in a way that's easy to miss in the task output. +This matters specifically for PQC. On AlmaLinux and RHEL 9, the `FIPS` policy alone includes no post-quantum key-exchange groups, but the OS still ships a built-in `PQ.pmod` that adds `mlkem768x25519-sha256` and other ML-KEM groups when combined as `FIPS:PQ`. This was confirmed against a real AlmaLinux 9.8 host, where `sshd -T` only showed `mlkem768x25519-sha256` in the effective `KexAlgorithms` after switching from `FIPS` to `FIPS:PQ`. AlmaLinux and RHEL 10 ship PQC key exchange in `FIPS` already, so the combination is not needed there. -Finally, once `sshd` has reloaded, the existing `sshd -T` check is extended to flag any requested algorithm that still isn't showing up in the effective config, so you find out before Phase 2 tries (and fails) to reconnect. +Before ever calling `update-crypto-policies --set`, the playbook lists whatever `*.pmod` files exist under both module directories on that host, the control node in Phase 0 and the target in Phase 1. If `ssh_key_rotation_crypto_policy_setting` names a module that is not present, it fails before making any change, rather than letting `update-crypto-policies` silently ignore an unknown module name or fail in a way that is easy to miss in the task output. ### Example: enabling a PQC key-exchange algorithm @@ -257,69 +326,79 @@ ansible-playbook krameff.ssh_key_rotation.rotate \ -e ssh_key_rotation_crypto_policy_setting=FIPS:PQ ``` -## How It Works +## Safety model + +The point of this playbook is that you should not be able to lock yourself out by running it. That comes down to a handful of rules it never breaks. + +**Nothing touches a live host until the new key has passed local checks.** Phase 0 verifies the new key's type, its strength, and that it genuinely pairs with the private key you gave it. -### Safety model +**Phase 1 confirms the target will actually accept the key type.** It reads the effective `PubkeyAcceptedAlgorithms` from `sshd -T`, after any crypto-policy has been applied, and fails the host before Phase 2 runs if there is no real signature algorithm for the new key's type. A key type can pass Phase 0's local checks and still be rejected by a target's crypto-policy, so this catches the problem before the old key is touched. -The whole point of this playbook is that you should never be able to lock yourself out by running it. That comes down to a few rules it never breaks: +**The new key is proven to work before anything old is removed.** Phase 2 opens by resetting the connection and reconnecting with the new key. If that fails, the playbook aborts on that host and none of the cleanup tasks run. -- Nothing is validated against a live host until Phase 0 has already checked the new key locally - its type, its strength, and that it genuinely pairs with the private key you gave it. -- Phase 1 confirms the target's *actual* effective `PubkeyAcceptedAlgorithms` (via `sshd -T`, after any crypto-policy is applied) includes a real signature algorithm for the new key's type, and fails the host before Phase 2 runs if it doesn't. A key type can pass Phase 0's local checks and still be rejected by a target's crypto-policy (RHEL/Fedora `FIPS` mode drops `ssh-ed25519`, for example) - this check catches that before the old key is ever touched. -- The new key is installed and proven to work before anything old is touched. Phase 2 opens by resetting the connection and reconnecting with the *new* key; if that fails, the playbook aborts on that host and none of the cleanup tasks run. Resetting the connection first matters if your `ansible.cfg` enables SSH `ControlPersist`: that multiplexes connections by host/port/user, not by identity file, so without the reset, Phase 2 could otherwise silently ride on Phase 1's still-open connection instead of genuinely testing the new key. -- Every `sshd_config` change is validated with `sshd -t` before it's written. -- Configuration is applied with a reload, never a restart, so sessions already open stay open. -- `sshd_config` and `authorized_keys` are both backed up before every edit, so you can always roll back by hand. -- Phase 2's cleanup (removing the old key, disabling legacy auth) runs inside an Ansible `block`/`rescue`. If anything in it fails partway through, `rescue` automatically restores `authorized_keys` and `sshd_config` from the backups just taken, reloads sshd, re-confirms connectivity, and fails with a clear message - all on the same still-open connection, before it could be lost. A reload that itself fails does not abort the rollback: the restored files are already on disk and sshd reads `authorized_keys` per connection, so the rollback finishes and the failure message tells you to reload sshd by hand. +Resetting the connection first matters if your `ansible.cfg` enables SSH `ControlPersist`. That multiplexes connections by host, port and user, not by identity file, so without the reset, Phase 2 could silently ride on Phase 1's still-open connection instead of genuinely testing the new key. + +**Every `sshd_config` change is validated with `sshd -t` before it is written.** + +**Configuration is applied with a reload, never a restart,** so sessions already open stay open. + +**`sshd_config` and `authorized_keys` are backed up before every edit,** so you can always roll back by hand. + +**Phase 2's cleanup runs inside a `block`/`rescue`.** If removing the old key or disabling legacy auth fails partway through, `rescue` restores `authorized_keys` and `sshd_config` from the backups just taken, reloads sshd, re-confirms connectivity, and fails with a clear message. All of that happens on the same still-open connection, before it could be lost. + +A reload that itself fails does not abort the rollback. The restored files are already on disk and sshd reads `authorized_keys` per connection, so the rollback finishes and the failure message tells you to reload sshd by hand. + +## What each phase does ### Phase 0: validate locally -1. Assert `old_private_key`, `new_private_key`, `old_public_key_file`, `new_public_key_file` are set -2. Inspect `new_public_key_file`'s type, size, and fingerprint -3. Reject weak/deprecated types and undersized RSA keys -4. Confirm `new_private_key` and `new_public_key_file` are a matching pair +1. Assert `old_private_key`, `new_private_key`, `old_public_key_file` and `new_public_key_file` are set. +2. Inspect the type, size and fingerprint of `new_public_key_file`. +3. Reject weak or deprecated types and undersized RSA keys. +4. Confirm `new_private_key` and `new_public_key_file` are a matching pair. ### Phase 1: install the new key -1. Determine the sshd service name for the OS family (Debian uses `ssh`, others use `sshd`) -2. Back up `authorized_keys` (if it exists) -3. Add the new public key to `authorized_keys`, keeping the old key in place for now -4. Enable `PubkeyAuthentication` in `sshd_config` (backed up first) -5. Make sure `AuthorizedKeysFile` points at the default location (backed up first) -6. If requested, append PQC/hybrid algorithms to `sshd_config` and/or apply a RHEL/Fedora crypto-policy (see [PQC Algorithm Negotiation](#pqc-algorithm-negotiation)) -7. Look for drop-in config files and `Match` blocks that might quietly override what was just set -8. Apply the sshd configuration changes -9. Extract the effective `PubkeyAcceptedAlgorithms`/`KexAlgorithms` from `sshd -T` and **fail the host** if the new key's real signature algorithm isn't among them (see [Key Validation](#key-validation-pqc-readiness)); warn, informationally, for any requested PQC algorithm that's still missing +1. Determine the sshd service name for the OS family. Debian uses `ssh`, others use `sshd`. +2. Back up `authorized_keys`, if it exists. +3. Add the new public key to `authorized_keys`, keeping the old key in place for now. +4. Enable `PubkeyAuthentication` in `sshd_config`, backing it up first. +5. Make sure `AuthorizedKeysFile` points at the default location, backing up first. +6. If requested, append PQC or hybrid algorithms to `sshd_config` and apply a RHEL/Fedora crypto-policy. +7. Look for drop-in config files and `Match` blocks that might quietly override what was just set. +8. Apply the sshd configuration changes. +9. Read the effective `PubkeyAcceptedAlgorithms` and `KexAlgorithms` from `sshd -T`, and fail the host if the new key's real signature algorithm is not among them. Any requested PQC algorithm that is still missing produces an informational warning. ### Phase 2: verify and clean up -1. Reset the connection, so nothing below can ride on a `ControlPersist` session left open from Phase 1 -2. Reconnect and gather facts with the new private key - this is the authentication gate -3. Ping the host to confirm the new key works -4. Assert the new key actually authenticated before doing anything else -5. From here on, steps 6-10 run inside a `block`/`rescue`: if any of them fail, `rescue` automatically restores `authorized_keys` and `sshd_config` from the backups taken below, reloads sshd (best effort - a failed reload is reported, not fatal, since the restored files are already in place), re-confirms connectivity, and fails with a clear message instead of leaving the host half-changed -6. Back up `authorized_keys`, then remove the old public key from it -7. Optionally make `authorized_keys` exclusive to the new key -8. Disable legacy auth methods (password, keyboard-interactive) if requested, backing up `sshd_config` first -9. Apply the final sshd configuration -10. One last connectivity check +1. Reset the connection, so nothing below can ride on a `ControlPersist` session left open from Phase 1. +2. Reconnect and gather facts with the new private key. This is the authentication gate. +3. Ping the host to confirm the new key works. +4. Assert the new key really authenticated before doing anything else. +5. Steps 6 to 10 run inside a `block`/`rescue`. If any of them fail, the backups taken below are restored, sshd is reloaded on a best-effort basis, connectivity is re-confirmed, and the play fails with a clear message rather than leaving the host half-changed. +6. Back up `authorized_keys`, then remove the old public key from it. +7. Optionally make `authorized_keys` exclusive to the new key. +8. Disable password and keyboard-interactive auth if requested, backing up `sshd_config` first. +9. Apply the final sshd configuration. +10. Run one last connectivity check. -## Role Reference +## Role reference -All the actual logic lives in the `ssh_key_rotation` role (`roles/ssh_key_rotation/`), following the standard Ansible role layout: +All the logic lives in the `ssh_key_rotation` role, under `roles/ssh_key_rotation/`, following the standard Ansible role layout: | Path | Purpose | |------|---------| | `defaults/main.yml` | Every optional variable in this README, with its default value | | `handlers/main.yml` | The single `Reload sshd` handler, shared by the install and verify stages | | `tasks/validate.yml` | Phase 0: local pre-flight validation, no remote connections | -| `tasks/install.yml` | Phase 1: connect with the OLD key, install the NEW key, prepare sshd | -| `tasks/verify.yml` | Phase 2: reconnect with the NEW key to prove it works, then remove the OLD key/legacy auth | -| `tasks/manage_crypto_policy.yml` | RHEL/Fedora crypto-policy management, shared via `include_tasks` by both the validate (control node) and install (target) stages | -| `meta/main.yml` | Role metadata (supported platforms, minimum Ansible version) | +| `tasks/install.yml` | Phase 1: connect with the old key, install the new key, prepare sshd | +| `tasks/verify.yml` | Phase 2: reconnect with the new key to prove it works, then remove the old key and legacy auth | +| `tasks/manage_crypto_policy.yml` | RHEL/Fedora crypto-policy management, included by both the validate and install stages | +| `meta/main.yml` | Role metadata: supported platforms, minimum Ansible version | -The role has no single `tasks/main.yml` entry point, because each stage authenticates differently (validate runs locally, install uses the OLD key, verify uses the NEW key) - it must be included with an explicit `tasks_from`. +There is deliberately no `tasks/main.yml` entry point, because each stage authenticates differently. Validate runs locally, install uses the old key, and verify uses the new key. The role must be included with an explicit `tasks_from`. -`playbooks/rotate.yml` is the entry point that ties the three stages together as three separate plays, since each needs a different host/connection context: +`playbooks/rotate.yml` is the entry point that ties the three stages together as three separate plays, since each needs a different host and connection context: ```yaml - hosts: localhost @@ -339,100 +418,96 @@ The role has no single `tasks/main.yml` entry point, because each stage authenti - ansible.builtin.include_role: {name: ssh_key_rotation, tasks_from: verify} ``` -Three-phase SSH key rotation with safety gates. - -**Hosts pattern:** `rotate` (define this group in your inventory) +## Troubleshooting -**Example inventory** (`inventory.ini`): +### Phase 0 validation failures -```ini -[rotate] -prod-web-01 ansible_host=10.0.1.10 -prod-web-02 ansible_host=10.0.1.11 -prod-db-01 ansible_host=10.0.2.10 -``` +These all happen before any host is touched, so there is no risk of a lockout while you fix them. -## Troubleshooting +**"is in ssh_key_rotation_reject_key_types"** - the new key's type, DSA for example, is explicitly blocked. Generate a new key with a stronger algorithm. -### Phase 0 validation failures +**"not in ssh_key_rotation_accepted_key_types or ssh_key_rotation_pqc_key_types"** - the type is not recognised. Either generate an accepted key type, or add the type to one of those two lists. -These all happen before any host is touched, so there's no risk of a lockout while you fix them: +**"RSA key; minimum accepted is ... bits"** - regenerate with `ssh-keygen -t rsa -b 4096`, or switch to `ed25519`. -- **"is in ssh_key_rotation_reject_key_types"** - the new key's type (e.g. DSA) is explicitly blocked; generate a new key with a stronger algorithm -- **"not in ssh_key_rotation_accepted_key_types or ssh_key_rotation_pqc_key_types"** - the type isn't recognized; either generate an accepted key type or add the type to `ssh_key_rotation_accepted_key_types`/`ssh_key_rotation_pqc_key_types` -- **"RSA key; minimum accepted is ... bits"** - regenerate with `ssh-keygen -t rsa -b 4096` or switch to `ed25519` -- **"does not match new_public_key_file"** - `new_private_key` and `new_public_key_file` aren't actually a matching pair; double-check both paths +**"does not match new_public_key_file"** - `new_private_key` and `new_public_key_file` are not actually a matching pair. Double-check both paths. ### "sshd -T ... does not include any signature algorithm for a ... key" -Phase 1 aborted before touching the old key: the target's effective `PubkeyAcceptedAlgorithms` (from `sshd -T`, after any crypto-policy is applied) doesn't include a real signature algorithm for your new key's type. The most common cause is a system-wide crypto-policy narrowing what's accepted - RHEL/Fedora's `FIPS` policy, for instance, only allows `ecdsa-sha2-*` and `rsa-sha2-*`, so an `ed25519` key will always be rejected there even though Phase 0's local checks consider it a perfectly good key type. Nothing has been broken; the old key is still in place. To fix it, either: +Phase 1 aborted before touching the old key. The target's effective `PubkeyAcceptedAlgorithms`, read from `sshd -T` after any crypto-policy was applied, does not include a real signature algorithm for your new key's type. -1. Generate a key type the target's policy actually accepts (check with `ssh sudo sshd -T | grep -i pubkeyacceptedalgorithms`), e.g. `ssh-keygen -t ecdsa -b 256`, or -2. Adjust the crypto-policy itself via `ssh_key_rotation_manage_crypto_policy`/`ssh_key_rotation_crypto_policy_setting` if the type you want should be allowed. +The most common cause is a system-wide crypto-policy narrowing what is accepted. RHEL and Fedora's `FIPS` policy, for instance, allows only `ecdsa-sha2-*` and `rsa-sha2-*`, so an `ed25519` key will always be rejected there even though Phase 0's local checks consider it a perfectly good key type. -This check exists because an earlier version of this playbook only did a substring search across the *entire* `sshd -T` output for the key type name, which produced false negatives (e.g. a `hostkey /etc/ssh/ssh_host_ed25519_key` line would satisfy a check for `ed25519`, even when `PubkeyAcceptedAlgorithms` didn't include `ssh-ed25519` at all) and let Phase 2 go on to remove the old key regardless. See [CHANGELOG.md](CHANGELOG.md) for details. +Nothing has been broken and the old key is still in place. To fix it, either: + +1. Generate a key type the target's policy accepts. Check with `ssh sudo sshd -T | grep -i pubkeyacceptedalgorithms`, then for example `ssh-keygen -t ecdsa -b 256`. +2. Adjust the crypto-policy itself via `ssh_key_rotation_manage_crypto_policy` and `ssh_key_rotation_crypto_policy_setting`, if the type you want should be allowed. + +This check exists because an earlier version of the playbook did a substring search across the entire `sshd -T` output for the key type name. That produced false negatives: a `hostkey /etc/ssh/ssh_host_ed25519_key` line would satisfy a check for `ed25519` even when `PubkeyAcceptedAlgorithms` did not include `ssh-ed25519` at all, and Phase 2 would go on to remove the old key regardless. See [CHANGELOG.md](CHANGELOG.md) for details. ### "New key did not authenticate" -Phase 2 couldn't connect with the new key. A few things to check: +Phase 2 could not connect with the new key. Things to check: -1. **Key paths are correct** - double-check `-e new_private_key` and `-e new_public_key_file` -2. **Key formats match** - the private key type has to match the public key (both ed25519, both rsa, etc.) -3. **File permissions** - the new private key needs to be readable by the Ansible user -4. **Public key was installed** - Phase 1 has to have completed successfully; check that the key is actually in `~/.ssh/authorized_keys` on the target -5. **Target's crypto-policy accepts the key type** - see "sshd -T ... does not include any signature algorithm" above; Phase 1 should already have caught this, but double-check with `sudo sshd -T | grep -i pubkeyacceptedalgorithms` on the target +1. **Key paths are correct.** Double-check `-e new_private_key` and `-e new_public_key_file`. +2. **Key formats match.** The private key type has to match the public key: both ed25519, both rsa, and so on. +3. **File permissions.** The new private key needs to be readable by the Ansible user. +4. **The public key was installed.** Phase 1 must have completed successfully. Check the key is actually in `~/.ssh/authorized_keys` on the target. +5. **The target's crypto-policy accepts the key type.** See the section above. Phase 1 should already have caught this, but confirm with `sudo sshd -T | grep -i pubkeyacceptedalgorithms` on the target. ### "Drop-in sshd config files found" Phase 1 spotted override files in `/etc/ssh/sshd_config.d/`. Worth reviewing: -1. See what's there: `ansible all -i inventory.ini -m ansible.builtin.find -a "paths=/etc/ssh/sshd_config.d patterns='*.conf'" -b` -2. Make sure none of them re-enable `PasswordAuthentication yes` or similar -3. If needed, update the drop-ins by hand before running Phase 2, or pass `ssh_key_rotation_make_exclusive=false` to keep the old key active a bit longer +1. See what is there: `ansible all -i inventory.ini -m ansible.builtin.find -a "paths=/etc/ssh/sshd_config.d patterns='*.conf'" -b` +2. Make sure none of them re-enable `PasswordAuthentication yes` or similar. +3. If needed, update the drop-ins by hand before running Phase 2, or pass `ssh_key_rotation_make_exclusive=false` to keep the old key active a little longer. ### PQC algorithms not negotiating -If Phase 1's post-reload check warns that `sshd -T` doesn't mention a requested PQC algorithm, or Phase 2 fails to authenticate with a PQC-type key, work through these in order: +If Phase 1's post-reload check warns that `sshd -T` does not mention a requested PQC algorithm, or Phase 2 fails to authenticate with a PQC-type key, work through these in order: -1. Confirm the control node's ssh binary actually supports the algorithms you asked for (Phase 0 already checks this, but `ssh -Q kex` / `ssh -Q key-sig` will tell you directly) -2. Confirm the target's sshd build supports them too: `sshd -T | grep -i kexalgorithms` -3. If `ssh_key_rotation_manage_crypto_policy` is set on a RHEL/Fedora host, check the policy actually changed: `update-crypto-policies --show` -4. Look for a `50-redhat.conf` or other drop-in overriding your settings (see "Drop-in sshd config files found" above) +1. Confirm the control node's ssh binary supports the algorithms you asked for. Phase 0 already checks this, but `ssh -Q kex` and `ssh -Q key-sig` will tell you directly. +2. Confirm the target's sshd build supports them too: `sshd -T | grep -i kexalgorithms`. +3. If `ssh_key_rotation_manage_crypto_policy` is set on a RHEL or Fedora host, check the policy actually changed: `update-crypto-policies --show`. +4. Look for a `50-redhat.conf` or other drop-in overriding your settings. ### "Desired crypto policy ... needs module(s) ... that were not found" -`ssh_key_rotation_crypto_policy_setting` named a `BASE:MODULE` combination (e.g. `FIPS:PQ`) but that host has no `MODULE.pmod` file under `/usr/share/crypto-policies/policies/modules/` or `/etc/crypto-policies/policies/modules/`. This is a hard stop *before* `update-crypto-policies --set` is ever called, on the control node (Phase 0) or the target (Phase 1), so nothing has changed on that host yet. To fix it: +`ssh_key_rotation_crypto_policy_setting` named a `BASE:MODULE` combination such as `FIPS:PQ`, but that host has no matching `MODULE.pmod` file under `/usr/share/crypto-policies/policies/modules/` or `/etc/crypto-policies/policies/modules/`. + +This is a hard stop before `update-crypto-policies --set` is ever called, on the control node in Phase 0 or the target in Phase 1, so nothing has changed on that host. To fix it: -1. List what's actually available: `ssh ls /usr/share/crypto-policies/policies/modules/*.pmod /etc/crypto-policies/policies/modules/*.pmod` -2. Check for typos in the module name, or drop a custom `MODULE.pmod` into `/etc/crypto-policies/policies/modules/` if you need one that doesn't ship with the OS -3. See [Combining a base policy with a subpolicy module](#combining-a-base-policy-with-a-subpolicy-module-basemodule) above for why `FIPS:PQ` is the combination most people want on RHEL/AlmaLinux 9 +1. List what is actually available: `ssh ls /usr/share/crypto-policies/policies/modules/*.pmod /etc/crypto-policies/policies/modules/*.pmod` +2. Check for typos in the module name, or drop a custom `MODULE.pmod` into `/etc/crypto-policies/policies/modules/` if you need one the OS does not ship. +3. See [Combining a base policy with a subpolicy module](#combining-a-base-policy-with-a-subpolicy-module) for why `FIPS:PQ` is the combination most people want on RHEL and AlmaLinux 9. ### "Permission denied" on Phase 1 Check that: -1. The old private key is readable and correct -2. The remote user matches the one Ansible is actually using (run with `-vvv` to confirm) -3. The SSH keys aren't passphrase-protected (or use `SSH_ASKPASS` with `--ask-pass`) +1. The old private key is readable and correct. +2. The remote user matches the one Ansible is actually using. Run with `-vvv` to confirm. +3. The SSH keys are not passphrase-protected, or use `SSH_ASKPASS` with `--ask-pass`. ### Module execution fails with "Operation not permitted" on a hardened host -If Ansible fails while gathering facts or running any module (not this playbook specifically) with an error like `can't open file '.../AnsiballZ_setup.py': [Errno 1] Operation not permitted`, the target likely has `/tmp`, `/var/tmp`, and/or the user's home directory mounted `noexec` - a common CIS/STIG hardening baseline. Ansible's default mechanism copies each module to a remote temp file and executes it, which a `noexec` mount blocks outright. Either: +If Ansible fails while gathering facts or running any module, not just this playbook, with an error like `can't open file '.../AnsiballZ_setup.py': [Errno 1] Operation not permitted`, the target likely has `/tmp`, `/var/tmp` or the user's home directory mounted `noexec`. That is a common CIS and STIG hardening baseline. -1. Enable [pipelining](https://docs.ansible.com/ansible/latest/collections/ansible/builtin/ssh_connection.html#parameter-pipelining) (`pipelining = True` under the `[ssh_connection]` section of `ansible.cfg`, or `ANSIBLE_PIPELINING=True`), which streams most modules over stdin instead of writing them to disk, or -2. Point `remote_tmp` (`ansible_remote_tmp`) at a directory that's genuinely executable on that host, if one exists +Ansible's default mechanism copies each module to a remote temp file and executes it, which a `noexec` mount blocks outright. Either: + +1. Enable [pipelining](https://docs.ansible.com/ansible/latest/collections/ansible/builtin/ssh_connection.html#parameter-pipelining), with `pipelining = True` under `[ssh_connection]` in `ansible.cfg` or `ANSIBLE_PIPELINING=True`. This streams most modules over stdin instead of writing them to disk. +2. Point `remote_tmp` (`ansible_remote_tmp`) at a directory that is genuinely executable on that host, if one exists. Pipelining is the more robust fix, since some hardening baselines make every writable path `noexec`. ## Development -### Running locally with vagrant or containers - ```bash # Build the collection ansible-galaxy collection build . -# Test with a local VM (bring your own via vagrant/libvirt/podman) +# Test against a local VM, brought up however you like ansible-playbook playbooks/rotate.yml -i 127.0.0.1, \ -e old_private_key=~/.ssh/id_rsa \ -e new_private_key=~/.ssh/id_ed25519 \ @@ -440,6 +515,20 @@ ansible-playbook playbooks/rotate.yml -i 127.0.0.1, \ -e old_public_key_file=./id_rsa.pub ``` +## Contributing + +Issues and bug reports are welcome and genuinely useful. + +Pull requests are accepted from existing contributors only. If you would like to become one, email . + +See [CONTRIBUTING.md](CONTRIBUTING.md) for the full detail. + +The short version for contributors: + +- Playbooks pass `ansible-lint` and both Molecule scenarios +- Changes preserve the safety model, meaning the new key is proven before the old key is removed +- README.md and CHANGELOG.md are updated to match + ## License MIT @@ -447,11 +536,3 @@ MIT ## Support For issues and feature requests, see the [GitHub repository](https://github.com/krameff/ssh_key_rotation). - -## Contributing - -Contributions are welcome. Please make sure: - -- Playbooks pass `ansible-lint` -- Changes preserve the safety model (new key proven before old key removed) -- Documentation is updated to match diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..fe6b59e --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,29 @@ +# Security Policy + +## Reporting a vulnerability + +Please do not open a public issue for a security problem. + +Report it privately through [GitHub Security Advisories](https://github.com/krameff/ssh_key_rotation/security/advisories/new), or by email to . + +We will acknowledge your report within five working days and keep you updated as we work on it. If you would like credit in the advisory and changelog, say so and tell us how you would like to be named. + +## What counts as a security issue here + +This collection edits `authorized_keys` and `sshd_config` on live hosts, so the interesting failures are availability failures as much as they are confidentiality ones. Please report privately, rather than as an ordinary bug: + +- **Any path that can leave a host unreachable.** If a run can remove or invalidate the old key without the new key having been proven to work, that is the most serious class of bug this project has, whatever the trigger. +- **A rollback that does not roll back.** The `block`/`rescue` in the verify stage is the last line of defence. If it can fail to restore `authorized_keys` or `sshd_config`, or can be skipped, we want to know privately. +- **A safety gate that can be bypassed.** For example, a case where Phase 2 authenticates over a connection left open by Phase 1 rather than genuinely testing the new key, or where `sshd -t` validation is skipped before a write. +- **Private key material leaking.** Into task output, a fact, a log, a backup file with loose permissions, or the built collection tarball. +- **A weakening of what the host will accept.** Anything that leaves `sshd_config` more permissive than the run asked for, such as password authentication surviving when it was meant to be disabled, or an algorithm list being replaced rather than appended to. + +Ordinary bugs, including a rotation that fails cleanly and leaves the host reachable on the old key, can go in a [public issue](https://github.com/krameff/ssh_key_rotation/issues). + +## Supported versions + +Security fixes are made against the latest released version. Please confirm the problem still reproduces there before reporting. + +## Scope + +This policy covers the collection itself: the role, its playbooks and its plugins. Vulnerabilities in OpenSSH, Ansible, or a target operating system should go to those projects, though we are glad to hear about anything that changes how this collection ought to behave in response. diff --git a/images/ssh-key-rotation-lockup-dark.svg b/images/ssh-key-rotation-lockup-dark.svg new file mode 100644 index 0000000..6515707 --- /dev/null +++ b/images/ssh-key-rotation-lockup-dark.svg @@ -0,0 +1,15 @@ + + SSH key rotation — a Krameff collection + + + + + + + + + + + + + diff --git a/images/ssh-key-rotation-lockup-light.svg b/images/ssh-key-rotation-lockup-light.svg new file mode 100644 index 0000000..5c8533c --- /dev/null +++ b/images/ssh-key-rotation-lockup-light.svg @@ -0,0 +1,15 @@ + + SSH key rotation — a Krameff collection + + + + + + + + + + + + + From a7ba61ff900147fa68c1db25bb6e6fbd837faaa4 Mon Sep 17 00:00:00 2001 From: Mark Bolwell Date: Wed, 5 Aug 2026 11:39:10 +0100 Subject: [PATCH 09/18] Updated core files and wording --- .pre-commit-config.yaml | 16 ++++++++++++++-- CONTRIBUTING.md | 21 ++++++++++++++------- DEVELOPMENT.md | 30 ++++++++++++++++++++++++------ 3 files changed, 52 insertions(+), 15 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fa247a9..afd8562 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,7 +4,13 @@ ##### Inherited ci: autofix_prs: false - skip: [detect-aws-credentials, ansible-lint] + skip: [ansible-lint] + +# Hook repos may pin a language_version newer than the interpreter available here +# (ansible-lint v26.3.0 asks for python3.14). Pin to the venv's Python so hook +# environments build against the same interpreter ansible-core is installed under. +default_language_version: + python: python3.12 repos: - repo: https://github.com/pre-commit/pre-commit-hooks @@ -13,6 +19,9 @@ repos: # Safety - id: detect-aws-credentials name: Detect AWS Credentials + # Without this the hook exits 2 on any machine that has no ~/.aws/credentials + # to compare against, which is a tooling error rather than a finding. + args: [--allow-missing-credentials] - id: detect-private-key name: Detect Private Keys @@ -54,8 +63,11 @@ repos: - id: ansible-lint name: Ansible-lint description: This hook runs ansible-lint. - entry: python3 -m ansiblelint --force-color site.yml -c .ansible-lint + # This collection has no site.yml. rotate.yml is only the entry point, so lint + # the whole repo to cover roles/ssh_key_rotation too. + entry: python3 -m ansiblelint --force-color -c .ansible-lint language: python + language_version: python3.12 # do not pass files to ansible-lint, see: # https://github.com/ansible/ansible-lint/issues/611 pass_filenames: false diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 69dc2d5..31a4cff 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -4,7 +4,7 @@ Thanks for your interest in improving this collection. This document covers how For deeper detail on project layout, adding modules or filters, and publishing releases, see [DEVELOPMENT.md](DEVELOPMENT.md). -NOTE: due to the number of open-source projects we work with and amount of PRS etc that have not been properly scoped. We only accept PRs from contributors. We will take all the information from issues and bug reports. +NOTE: due to the number of open-source projects we work with and amount of PRS etc that have not been properly scoped. We only accept PRs from contributors. We will take all the information from issues and bug reports. If you are interested in becoming a contributor, please drop us an email at . ## The one rule that matters most @@ -28,9 +28,13 @@ git clone https://github.com/krameff/ssh_key_rotation cd ssh_key_rotation python -m venv .venv && source .venv/bin/activate -pip install "ansible>=2.18" ansible-lint "molecule>=25.0" "molecule-plugins[docker]" +pip install "ansible>=2.18" ansible-lint pre-commit "molecule>=25.0" "molecule-plugins[docker]" ansible-galaxy install -r requirements.yml + +# Install the git hooks. Do not skip this: it is what stops a private key or a +# secret reaching a commit in the first place +pre-commit install ``` Molecule needs a working Docker or Podman socket. The scenarios pull `geerlingguy` CI images for Ubuntu 22.04, Rocky Linux 9 and Rocky Linux 10. @@ -62,14 +66,17 @@ The role has no working `tasks/main.yml` entry point by design, because each sta These are the same checks CI runs, so running them locally saves a round trip. ```bash -# Lint the whole repo, not just the playbook, so the role is covered too -ansible-lint +# Everything the hooks cover: secret and private key detection, whitespace, +# YAML lint, and ansible-lint. Runs automatically on commit once installed +pre-commit run --all-files -# Syntax check the entry point +# Not covered by a hook, so run it separately ansible-playbook playbooks/rotate.yml --syntax-check ``` -`ansible-lint` runs on the `production` profile. Do not add `# noqa` comments to silence a rule without explaining why in the same commit. +`ansible-lint` runs on the `production` profile, and lints the whole repository rather than just `playbooks/rotate.yml`, since that is only the entry point and all the logic lives in the role. Do not add `# noqa` comments to silence a rule without explaining why in the same commit. + +The hooks are the last line of defence against committing key material, which is why `pre-commit install` is not optional here. Pull requests are also checked by pre-commit.ci, which runs everything except `ansible-lint`. That one needs `ansible.posix` installed, so it runs in the CI workflow instead. ### Functional tests @@ -149,7 +156,7 @@ Do not bump the version in `galaxy.yml` in a pull request. Releases are cut sepa ## Pull request checklist -- [ ] `ansible-lint` passes with no new warnings +- [ ] `pre-commit run --all-files` passes, ansible-lint included - [ ] `ansible-playbook playbooks/rotate.yml --syntax-check` passes - [ ] `molecule test -s default` and `molecule test -s rollback` pass, or you have explained why they could not be run - [ ] New behaviour is covered by a test that fails without the change diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 0afddc4..c3a222f 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -68,15 +68,20 @@ python -m venv .venv && source .venv/bin/activate pip install "ansible>=2.18" # Testing tools -pip install ansible-lint "molecule>=25.0" "molecule-plugins[docker]" +pip install ansible-lint pre-commit "molecule>=25.0" "molecule-plugins[docker]" # Collection dependencies ansible-galaxy install -r requirements.yml +# Install the git hooks +pre-commit install + # Sanity check the collection structure ansible-galaxy collection build . ``` +`pre-commit install` writes to `.git/hooks/`, which is not tracked, so every clone needs it run once. + ## Adding code ### A new playbook @@ -100,16 +105,29 @@ ansible-galaxy collection build . ### Lint and syntax -These are the two checks CI runs on every push and pull request. - ```bash -# Lint the whole repo. rotate.yml is only the entry point, so linting it alone misses the role -ansible-lint +# Every hook: secret and private key detection, whitespace, YAML lint, ansible-lint +pre-commit run --all-files +# Not covered by a hook ansible-playbook playbooks/rotate.yml --syntax-check ``` -`ansible-lint` runs on the `production` profile, configured in `.ansible-lint`. +`ansible-lint` runs on the `production` profile, configured in `.ansible-lint`. It lints the whole repository, because `playbooks/rotate.yml` is only the entry point and linting it alone would miss the role. + +Hooks run automatically on commit once `pre-commit install` has been run. To bypass them for a work-in-progress commit, `git commit --no-verify`, but do not push a branch that has not passed them. + +### Where each check runs + +| Check | Where | +|-------|-------| +| All hooks except `ansible-lint` | pre-commit.ci, on pull requests | +| `ansible-lint`, syntax check | `.github/workflows/ci.yml` | +| Molecule scenarios | `.github/workflows/molecule.yml` | + +`ansible-lint` is in the `ci.skip` list in `.pre-commit-config.yaml` because pre-commit.ci's runner cannot install `ansible.posix`, so the hook could not resolve the collection there. It runs in the CI workflow instead, which does an `ansible-galaxy install -r requirements.yml` first. + +`.pre-commit-config.yaml` also pins `default_language_version` to Python 3.12, because the ansible-lint hook's manifest asks for an interpreter newer than most machines have installed, and pre-commit fails to build the hook environment at all without the override. ### Dry run From fa20016637b2fec5c5d14f9236ca411333a4846a Mon Sep 17 00:00:00 2001 From: Mark Bolwell Date: Wed, 5 Aug 2026 11:39:49 +0100 Subject: [PATCH 10/18] permissions update and attestation added --- .github/workflows/release.yml | 36 +++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 419d0d5..10fedf1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -9,9 +9,18 @@ on: # Lets the build be exercised without cutting a tag; publishing stays gated on the tag. workflow_dispatch: +# Default the whole workflow to read-only. The job below widens this to exactly what it +# needs. This job holds GALAXY_API_KEY, so it should not also inherit a writable token. +permissions: + contents: read + jobs: build-and-publish: runs-on: ubuntu-latest + permissions: + contents: write # create the GitHub Release and attach the tarball + id-token: write # Sigstore keyless signing for the provenance attestation + attestations: write # record the attestation against this repository steps: - name: Check out repository uses: actions/checkout@v7 @@ -41,12 +50,39 @@ jobs: - name: Build the collection artifact run: ansible-galaxy collection build --output-path ./dist + # Ties the tarball to this repository, commit and workflow run. Keyless via Sigstore, + # so there is no signing key to hold. Verify with: + # gh attestation verify --repo krameff/ssh_key_rotation + - name: Attest build provenance + uses: actions/attest-build-provenance@v4 + with: + subject-path: dist/*.tar.gz + + # Kept for workflow_dispatch runs, where no Release is created. Expires with the run. - name: Upload the artifact to the workflow run uses: actions/upload-artifact@v5 with: name: collection-tarball path: dist/*.tar.gz + # Galaxy is the normal install route, but it serves no signature or provenance for + # community collections. Attaching the same tarball here gives anyone who wants to + # check where it came from something they can actually fetch and verify. + - name: Create the GitHub Release + if: startsWith(github.ref, 'refs/tags/v') + env: + GH_TOKEN: ${{ github.token }} + run: | + gh release create "${GITHUB_REF_NAME}" dist/*.tar.gz \ + --title "${GITHUB_REF_NAME}" \ + --notes "See [CHANGELOG.md](https://github.com/${GITHUB_REPOSITORY}/blob/${GITHUB_REF_NAME}/CHANGELOG.md). + + Verify this tarball came from this repository: + + \`\`\` + gh attestation verify krameff-ssh_key_rotation-*.tar.gz --repo ${GITHUB_REPOSITORY} + \`\`\`" + - name: Publish to Ansible Galaxy if: startsWith(github.ref, 'refs/tags/v') env: From ac728290f693351be7775cd2f65307f718a68d03 Mon Sep 17 00:00:00 2001 From: Mark Bolwell Date: Wed, 5 Aug 2026 11:40:01 +0100 Subject: [PATCH 11/18] Changelog updated --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 42f24af..5d7c03e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,11 @@ - Release workflow: tagging `v*` builds the collection and publishes it to Galaxy, needs a `GALAXY_API_KEY` secret. - Release workflow fails the build if the tag doesn't match the version in `galaxy.yml`. +- `CONTRIBUTING.md`, covering setup, the required checks and the pull request checklist. +- `SECURITY.md`, with a private disclosure route for anything that could leave a host unreachable. +- GitHub issue forms and a pull request template under `.github/`. +- Pre-commit configuration for secret detection, whitespace, YAML and ansible-lint. +- Releases now attach the collection tarball to a GitHub Release and attest its build provenance, so it can be traced back to this repository. ### Fixed From 9c1806ca1e37be1ae7b079b9f5a7e779e02d0e16 Mon Sep 17 00:00:00 2001 From: Mark Bolwell Date: Thu, 6 Aug 2026 19:17:24 +0100 Subject: [PATCH 12/18] Fix lockout bugs in the rollback path Signed-off-by: Mark Bolwell --- roles/ssh_key_rotation/tasks/install.yml | 554 ++++++++++++------ .../tasks/resolve_sshd_anchor.yml | 43 ++ .../tasks/resolve_target_identity.yml | 30 + roles/ssh_key_rotation/tasks/verify.yml | 515 ++++++++++++++-- 4 files changed, 907 insertions(+), 235 deletions(-) create mode 100644 roles/ssh_key_rotation/tasks/resolve_sshd_anchor.yml create mode 100644 roles/ssh_key_rotation/tasks/resolve_target_identity.yml diff --git a/roles/ssh_key_rotation/tasks/install.yml b/roles/ssh_key_rotation/tasks/install.yml index fe4a1bd..7e59e1f 100644 --- a/roles/ssh_key_rotation/tasks/install.yml +++ b/roles/ssh_key_rotation/tasks/install.yml @@ -11,201 +11,387 @@ ansible.builtin.set_fact: ssh_key_rotation_new_key_type: "{{ ssh_key_rotation_new_key_info.stdout | regex_search('\\(([^)]+)\\)\\s*$') | regex_replace('[()]', '') }}" +- name: Install | Resolve the target user's primary group + ansible.builtin.include_tasks: resolve_target_identity.yml + +# Decides, once, how this host will be configured. Drop-ins are used wherever sshd supports them, +# so that undoing this run means deleting a file rather than restoring the administrator's own +# sshd_config from a backup. Only hosts predating OpenSSH 8.2's Include support fall back to +# editing the main file. +- name: Install | Decide how this host's sshd will be configured + ansible.builtin.include_tasks: resolve_sshd_strategy.yml + - name: Install | Check whether authorized_keys already exists ansible.builtin.stat: path: "~{{ ssh_key_rotation_target_user }}/.ssh/authorized_keys" register: ssh_key_rotation_authorized_keys_stat +# owner/group are explicit, not left to `mode: preserve`: preserve copies the MODE only, never +# the ownership. These plays normally run with become, so the copy executes as root and the new +# file would otherwise be root-owned. sshd's StrictModes (on by default) ignores any +# authorized_keys not owned by the target user, which silently breaks key auth for that user. - name: Install | Back up authorized_keys before modifying it when: ssh_key_rotation_authorized_keys_stat.stat.exists ansible.builtin.copy: src: "~{{ ssh_key_rotation_target_user }}/.ssh/authorized_keys" - dest: "~{{ ssh_key_rotation_target_user }}/.ssh/authorized_keys.bak-{{ ansible_date_time.iso8601_basic_short }}" + dest: "~{{ ssh_key_rotation_target_user }}/.ssh/authorized_keys.bak-{{ ansible_facts.date_time.iso8601_basic_short }}" remote_src: true mode: preserve + owner: "{{ ssh_key_rotation_target_user }}" + group: "{{ ssh_key_rotation_target_group }}" + register: ssh_key_rotation_install_ak_backup -- name: Install | Install the new (PwC) public key in authorized_keys - ansible.posix.authorized_key: - user: "{{ ssh_key_rotation_target_user }}" - key: "{{ lookup('file', new_public_key_file) }}" - state: present - exclusive: false # keep the old key for now - we still need it - -- name: Install | Ensure public key authentication is enabled - ansible.builtin.lineinfile: - path: /etc/ssh/sshd_config - regexp: '^#?\s*PubkeyAuthentication' - line: 'PubkeyAuthentication yes' - validate: 'sshd -t -f %s' - backup: true - become: true - notify: Reload sshd - -- name: Install | Ensure AuthorizedKeysFile points at the default location - ansible.builtin.lineinfile: - path: /etc/ssh/sshd_config - regexp: '^#?\s*AuthorizedKeysFile' - line: 'AuthorizedKeysFile .ssh/authorized_keys' - validate: 'sshd -t -f %s' - backup: true - become: true - notify: Reload sshd - -# Opt-in, empty by default; the + prefix appends to OpenSSH's compiled-in defaults so non-PQC clients can still fall back. -- name: Install | Enable PQC/hybrid key exchange algorithms in sshd_config - when: ssh_key_rotation_pqc_kex_algorithms | length > 0 - ansible.builtin.lineinfile: - path: /etc/ssh/sshd_config - regexp: '^#?\s*KexAlgorithms' - line: 'KexAlgorithms +{{ ssh_key_rotation_pqc_kex_algorithms | join(",") }}' - validate: 'sshd -t -f %s' - backup: true - become: true - notify: Reload sshd - -- name: Install | Enable PQC/hybrid public key algorithms in sshd_config - when: ssh_key_rotation_pqc_pubkey_algorithms | length > 0 - ansible.builtin.lineinfile: - path: /etc/ssh/sshd_config - regexp: '^#?\s*PubkeyAcceptedAlgorithms' - line: 'PubkeyAcceptedAlgorithms +{{ ssh_key_rotation_pqc_pubkey_algorithms | join(",") }}' - validate: 'sshd -t -f %s' - backup: true - become: true - notify: Reload sshd - -# Reuses pqc_pubkey_algorithms for host keys too, since the same signature algorithms usually apply to both user and host keys. -- name: Install | Enable PQC/hybrid host key algorithms in sshd_config - when: ssh_key_rotation_pqc_pubkey_algorithms | length > 0 - ansible.builtin.lineinfile: - path: /etc/ssh/sshd_config - regexp: '^#?\s*HostKeyAlgorithms' - line: 'HostKeyAlgorithms +{{ ssh_key_rotation_pqc_pubkey_algorithms | join(",") }}' - validate: 'sshd -t -f %s' - backup: true - become: true - notify: Reload sshd - -- name: Install | Enable PQC/hybrid CA signature algorithms in sshd_config - when: ssh_key_rotation_pqc_ca_signature_algorithms | length > 0 - ansible.builtin.lineinfile: - path: /etc/ssh/sshd_config - regexp: '^#?\s*CASignatureAlgorithms' - line: 'CASignatureAlgorithms +{{ ssh_key_rotation_pqc_ca_signature_algorithms | join(",") }}' - validate: 'sshd -t -f %s' - backup: true - become: true - notify: Reload sshd - -# Same opt-in crypto-policy handling as the validate stage's control-node block, detected via the tool rather than ansible_os_family. -# Shared with validate.yml's control-node equivalent; see tasks/manage_crypto_policy.yml. -- name: Install | Manage this host's system-wide crypto policy - ansible.builtin.include_tasks: manage_crypto_policy.yml +# Recorded on the host itself, not just as a registered variable. The verify stage runs in a +# separate play and may even be run standalone, and an undefined variable inside a rescue: aborts +# the rescue entirely - leaving a half-changed host with no rollback at all, which is worse than +# having none. See state_write.yml. +- name: Install | Record the pre-install state on the host + ansible.builtin.include_tasks: state_write.yml vars: - ssh_key_rotation_crypto_policy_context: "this host" - ssh_key_rotation_crypto_policy_no_tool_followup: "Falling back to the direct sshd_config algorithm edits above only." - -# update-crypto-policies only validates its own syntax, not the merged sshd_config, so re-validate here before the reload handler fires. -- name: Install | Validate sshd_config after crypto-policy check/apply - when: - - ssh_key_rotation_manage_crypto_policy | bool - - ssh_key_rotation_crypto_policies_tool.rc == 0 - ansible.builtin.command: sshd -t - become: true - changed_when: false - register: ssh_key_rotation_sshd_validate_after_crypto_policy - failed_when: ssh_key_rotation_sshd_validate_after_crypto_policy.rc != 0 - -# A drop-in (e.g. /etc/ssh/sshd_config.d/50-cloud-init.conf) can override the main file - if your hosts use drop-ins, write these settings there instead. -- name: Install | Detect any sshd_config drop-in files that could override settings - ansible.builtin.find: - paths: /etc/ssh/sshd_config.d - patterns: '*.conf' - become: true - register: ssh_key_rotation_sshd_dropins - failed_when: false - -- name: Install | Warn if drop-in config files are present - when: ssh_key_rotation_sshd_dropins.files | default([]) | length > 0 - ansible.builtin.debug: - msg: >- - Drop-in sshd config files found: {{ ssh_key_rotation_sshd_dropins.files | map(attribute='path') | list }}. - Verify none of them re-enable the settings this playbook is changing. - {{ "Note: 50-redhat.conf is the crypto-policy backend include - manage its " ~ - "algorithm settings via update-crypto-policies/manage_crypto_policy, not by " ~ - "hand-editing it directly." if ssh_key_rotation_sshd_dropins.files | map(attribute='path') - | select('search', '50-redhat.conf') | list | length > 0 else "" }} - -# lineinfile only edits the global section; a Match block (e.g. Match User ...) can override these settings and isn't inspected here. -- name: Install | Detect Match blocks in sshd_config that could override global settings - ansible.builtin.command: "grep -n '^[[:space:]]*Match' /etc/ssh/sshd_config" - become: true - register: ssh_key_rotation_sshd_match_blocks - changed_when: false - failed_when: false - -- name: Install | Warn if Match blocks are present - when: ssh_key_rotation_sshd_match_blocks.stdout_lines | default([]) | length > 0 - ansible.builtin.debug: - msg: >- - sshd_config contains Match block(s): {{ ssh_key_rotation_sshd_match_blocks.stdout_lines }}. - This playbook only edits the global section - settings inside a Match - block are NOT modified and could silently override what this playbook sets. - -- name: Install | Apply sshd changes now so the new key is usable in the verify stage - ansible.builtin.meta: flush_handlers - -- name: Install | Check the effective sshd configuration for the new key's algorithm - ansible.builtin.command: sshd -T - become: true - changed_when: false - failed_when: false - register: ssh_key_rotation_sshd_effective_config + ssh_key_rotation_state: + run_id: "{{ ansible_facts.date_time.iso8601_basic_short }}" + stage: install-started + target_user: "{{ ssh_key_rotation_target_user }}" + target_group: "{{ ssh_key_rotation_target_group }}" + authorized_keys: + backup: "{{ ssh_key_rotation_install_ak_backup.dest | default(none) }}" + existed_before: "{{ ssh_key_rotation_authorized_keys_stat.stat.exists }}" + uses_dropin: "{{ ssh_key_rotation_use_dropin }}" -# Extracted as real comma-split lists, not a whole-blob substring search (which false-matched "ed25519" against the unrelated hostkey path line). -- name: Install | Extract the effective PubkeyAcceptedAlgorithms as a list - when: ssh_key_rotation_sshd_effective_config.rc == 0 - ansible.builtin.set_fact: - ssh_key_rotation_effective_pubkey_algorithms: >- - {{ (ssh_key_rotation_sshd_effective_config.stdout - | regex_search('(?im)^pubkeyacceptedalgorithms\s+(\S+)$', '\1') - | default([], true) | first | default('')).split(',') }} - ssh_key_rotation_effective_kex_algorithms: >- - {{ (ssh_key_rotation_sshd_effective_config.stdout - | regex_search('(?im)^kexalgorithms\s+(\S+)$', '\1') - | default([], true) | first | default('')).split(',') }} - -# Hard failure, not a warning: stops before the verify stage could remove the old key if the target's crypto-policy would reject the new key entirely. -- name: Install | Fail if the target's sshd will not accept the new key's signature algorithm - when: ssh_key_rotation_sshd_effective_config.rc == 0 - ansible.builtin.assert: - that: >- - (ssh_key_rotation_key_type_signature_algorithms[ssh_key_rotation_new_key_type] | default([ssh_key_rotation_new_key_type | lower])) - | intersect(ssh_key_rotation_effective_pubkey_algorithms) | length > 0 - fail_msg: >- - sshd -T on this host reports PubkeyAcceptedAlgorithms={{ ssh_key_rotation_effective_pubkey_algorithms | join(',') }}, - which does not include any signature algorithm for a {{ ssh_key_rotation_new_key_type }} key - ({{ ssh_key_rotation_key_type_signature_algorithms[ssh_key_rotation_new_key_type] | default([ssh_key_rotation_new_key_type]) | join(',') }}). - This host's crypto-policy would reject the new key entirely. Generate a key type this host's - PubkeyAcceptedAlgorithms accepts (see README.md's Key Validation section), or adjust - pqc_pubkey_algorithms/manage_crypto_policy, before continuing. Aborting now, before - the verify stage would remove the old key - nothing has been broken. - success_msg: "Target sshd will accept the new {{ ssh_key_rotation_new_key_type }} key's signature algorithm." - -- name: Install | Determine which requested PQC algorithms are missing from the effective sshd config - when: - - ssh_key_rotation_sshd_effective_config.rc == 0 - - (ssh_key_rotation_pqc_kex_algorithms + ssh_key_rotation_pqc_pubkey_algorithms) | length > 0 - ansible.builtin.set_fact: - ssh_key_rotation_missing_pqc_algorithms: >- - {{ (ssh_key_rotation_pqc_kex_algorithms | difference(ssh_key_rotation_effective_kex_algorithms)) - + (ssh_key_rotation_pqc_pubkey_algorithms | difference(ssh_key_rotation_effective_pubkey_algorithms)) }} - -- name: Install | Warn if sshd's effective config doesn't appear to include the requested PQC algorithms - when: - - ssh_key_rotation_missing_pqc_algorithms is defined - - ssh_key_rotation_missing_pqc_algorithms | length > 0 - ansible.builtin.debug: - msg: >- - sshd -T on this host does not list: {{ ssh_key_rotation_missing_pqc_algorithms }}. - The verify stage may fail to negotiate these; review sshd_config/crypto-policy before proceeding. +# Everything below CHANGES the host: it adds the new key to authorized_keys, writes this role's +# sshd configuration, and may switch the system-wide crypto policy. Previously none of it was +# undone when a later task failed - so when the crypto-policy guard further down refused a key type +# the host was left carrying the new key and this role's config, while the run reported that +# "nothing has been broken". Access was indeed intact, but the host was not as we found it. +# +# The rescue below returns it to its pre-install state. It is safe to restore authorized_keys here +# in a way the verify stage cannot assume: this play is CONNECTED over the old key, so the +# credential being restored is demonstrably working. It is still probed rather than assumed. +- name: Install | Apply changes, with automatic rollback if any of them fails + block: + - name: Install | Install the new (PwC) public key in authorized_keys + ansible.posix.authorized_key: + user: "{{ ssh_key_rotation_target_user }}" + key: "{{ lookup('file', new_public_key_file) }}" + state: present + exclusive: false # keep the old key for now - we still need it + + # Settings this stage needs, rendered once and written either to a drop-in or - on pre-8.2 hosts - + # to a marked block in sshd_config. + # + # AuthorizedKeysFile is deliberately NOT set. This role used to force it to `.ssh/authorized_keys`, + # which is harmless while it sits in the main file (an administrator's drop-in outranks it), but at + # the 01- prefix it would outrank a central key store such as + # `AuthorizedKeysFile /etc/ssh/authorized_keys/%u` and strip key access from every OTHER user on the + # host - while the rotated user, whose key was just written to ~/.ssh, kept working. The run would + # report success having locked out the rest of the estate. The value is read from `sshd -T` and + # asserted further down instead, so a host that keeps keys elsewhere fails loudly rather than being + # silently reconfigured. + # + # The PQC directives keep the `+` prefix, which appends to OpenSSH's compiled-in defaults so + # clients that do not speak PQC can still fall back to a classical algorithm. + - name: Install | Render this stage's sshd settings + ansible.builtin.set_fact: + ssh_key_rotation_install_settings: | + # Managed by krameff.ssh_key_rotation (install stage). Safe to remove; this role rewrites it. + PubkeyAuthentication yes + {% if ssh_key_rotation_pqc_kex_algorithms | length > 0 %} + KexAlgorithms +{{ ssh_key_rotation_pqc_kex_algorithms | join(",") }} + {% endif %} + {% if ssh_key_rotation_pqc_pubkey_algorithms | length > 0 %} + PubkeyAcceptedAlgorithms +{{ ssh_key_rotation_pqc_pubkey_algorithms | join(",") }} + HostKeyAlgorithms +{{ ssh_key_rotation_pqc_pubkey_algorithms | join(",") }} + {% endif %} + {% if ssh_key_rotation_pqc_ca_signature_algorithms | length > 0 %} + CASignatureAlgorithms +{{ ssh_key_rotation_pqc_ca_signature_algorithms | join(",") }} + {% endif %} + + - name: Install | Write this stage's settings to a drop-in + when: ssh_key_rotation_use_dropin | bool + ansible.builtin.include_tasks: sshd_dropin_write.yml + vars: + ssh_key_rotation_dropin_name: ssh-key-rotation.conf + ssh_key_rotation_dropin_key: install + ssh_key_rotation_dropin_content: "{{ ssh_key_rotation_install_settings }}" + + # Pre-8.2 fallback only. Anchored to the global section because a lineinfile regexp matching a + # keyword anywhere in the file replaces the LAST occurrence, and Match blocks sit at the END of + # sshd_config - so the previous code silently rewrote per-user policy instead of the global setting. + - name: Install | Fall back to a marked block in sshd_config on hosts without Include + when: not (ssh_key_rotation_use_dropin | bool) + ansible.builtin.blockinfile: + path: /etc/ssh/sshd_config + marker: "# {mark} krameff.ssh_key_rotation install" + # default(omit, true) turns an empty anchor into "parameter not supplied", so blockinfile + # falls back to its EOF default. At most one of the two is ever non-empty. + insertafter: "{{ ssh_key_rotation_block_after | default(omit, true) }}" + insertbefore: "{{ ssh_key_rotation_block_before | default(omit, true) }}" + block: "{{ ssh_key_rotation_install_settings }}" + validate: 'sshd -t -f %s' + backup: true + become: true + notify: Reload sshd + + # Same opt-in crypto-policy handling as the validate stage's control-node block, detected via the tool rather than ansible_os_family. + # Shared with validate.yml's control-node equivalent; see tasks/manage_crypto_policy.yml. + - name: Install | Manage this host's system-wide crypto policy + ansible.builtin.include_tasks: manage_crypto_policy.yml + vars: + ssh_key_rotation_crypto_policy_context: "this host" + ssh_key_rotation_crypto_policy_no_tool_followup: "Falling back to the direct sshd_config algorithm edits above only." + + # update-crypto-policies only validates its own syntax, not the merged sshd_config, so re-validate here before the reload handler fires. + - name: Install | Validate sshd_config after crypto-policy check/apply + when: + - ssh_key_rotation_manage_crypto_policy | bool + - ssh_key_rotation_crypto_policies_tool.rc == 0 + ansible.builtin.command: sshd -t + become: true + changed_when: false + register: ssh_key_rotation_sshd_validate_after_crypto_policy + failed_when: ssh_key_rotation_sshd_validate_after_crypto_policy.rc != 0 + + # A drop-in (e.g. /etc/ssh/sshd_config.d/50-cloud-init.conf) can override the main file - if your hosts use drop-ins, write these settings there instead. + - name: Install | Detect any sshd_config drop-in files that could override settings + ansible.builtin.find: + paths: /etc/ssh/sshd_config.d + patterns: '*.conf' + become: true + register: ssh_key_rotation_sshd_dropins + failed_when: false + + - name: Install | Warn if drop-in config files are present + when: ssh_key_rotation_sshd_dropins.files | default([]) | length > 0 + ansible.builtin.debug: + msg: >- + Drop-in sshd config files found: {{ ssh_key_rotation_sshd_dropins.files | map(attribute='path') | list }}. + Verify none of them re-enable the settings this playbook is changing. + {{ "Note: 50-redhat.conf is the crypto-policy backend include - manage its " ~ + "algorithm settings via update-crypto-policies/manage_crypto_policy, not by " ~ + "hand-editing it directly." if ssh_key_rotation_sshd_dropins.files | map(attribute='path') + | select('search', '50-redhat.conf') | list | length > 0 else "" }} + + # lineinfile only edits the global section; a Match block (e.g. Match User ...) can override these settings and isn't inspected here. + - name: Install | Detect Match blocks in sshd_config that could override global settings + ansible.builtin.command: "grep -n '^[[:space:]]*Match' /etc/ssh/sshd_config" + become: true + register: ssh_key_rotation_sshd_match_blocks + changed_when: false + failed_when: false + + - name: Install | Warn if Match blocks are present + when: ssh_key_rotation_sshd_match_blocks.stdout_lines | default([]) | length > 0 + ansible.builtin.debug: + msg: >- + sshd_config contains Match block(s): {{ ssh_key_rotation_sshd_match_blocks.stdout_lines }}. + This playbook only edits the global section - settings inside a Match + block are NOT modified and could silently override what this playbook sets. + + - name: Install | Apply sshd changes now so the new key is usable in the verify stage + ansible.builtin.meta: flush_handlers + + - name: Install | Check the effective sshd configuration for the new key's algorithm + ansible.builtin.command: sshd -T + become: true + changed_when: false + failed_when: false + register: ssh_key_rotation_sshd_effective_config + + # Extracted as real comma-split lists, not a whole-blob substring search (which false-matched "ed25519" against the unrelated hostkey path line). + - name: Install | Extract the effective PubkeyAcceptedAlgorithms as a list + when: ssh_key_rotation_sshd_effective_config.rc == 0 + ansible.builtin.set_fact: + ssh_key_rotation_effective_pubkey_algorithms: >- + {{ (ssh_key_rotation_sshd_effective_config.stdout + | regex_search('(?im)^pubkeyacceptedalgorithms\s+(\S+)$', '\1') + | default([], true) | first | default('')).split(',') }} + ssh_key_rotation_effective_kex_algorithms: >- + {{ (ssh_key_rotation_sshd_effective_config.stdout + | regex_search('(?im)^kexalgorithms\s+(\S+)$', '\1') + | default([], true) | first | default('')).split(',') }} + + # Hard failure, not a warning: stops before the verify stage could remove the old key if the target's crypto-policy would reject the new key entirely. + # Read, never written. This role installs the key into ~/.ssh/authorized_keys via + # ansible.posix.authorized_key, so if the host keeps keys somewhere else - a central store such as + # `AuthorizedKeysFile /etc/ssh/authorized_keys/%u` is common on hardened estates - the key lands + # somewhere sshd will not look and the rotation is silently ineffective. Forcing the value instead + # would be worse: at drop-in precedence it would redirect EVERY user on the host to ~/.ssh, locking + # out everyone whose keys live in the central store while the rotated user kept working. + - name: Install | Extract the effective AuthorizedKeysFile + when: ssh_key_rotation_sshd_effective_config.rc == 0 + ansible.builtin.set_fact: + ssh_key_rotation_effective_authorized_keys_file: >- + {{ (ssh_key_rotation_sshd_effective_config.stdout + | regex_search('(?im)^authorizedkeysfile\s+(.+)$', '\1') + | default([], true) | first | default('')) }} + + - name: Install | Fail if this host does not read keys from the user's own ~/.ssh/authorized_keys + when: + - ssh_key_rotation_sshd_effective_config.rc == 0 + - ssh_key_rotation_effective_authorized_keys_file | length > 0 + ansible.builtin.assert: + that: >- + ssh_key_rotation_effective_authorized_keys_file.split() + | select('search', '(^|/)\.ssh/authorized_keys') | list | length > 0 + fail_msg: >- + sshd on this host reports AuthorizedKeysFile={{ ssh_key_rotation_effective_authorized_keys_file }}, + which does not include the user's own ~/.ssh/authorized_keys. This role installs the new key + there, so sshd would never read it and the rotation would appear to succeed while the new key + did nothing. Point AuthorizedKeysFile at .ssh/authorized_keys for this user, or install the key + into the location above by other means, before rotating. Aborting before the verify stage could + remove the old key - nothing has been broken. + success_msg: >- + sshd reads keys from the user's own ~/.ssh/authorized_keys + ({{ ssh_key_rotation_effective_authorized_keys_file }}), which is where this role installs them. + + - name: Install | Fail if the target's sshd will not accept the new key's signature algorithm + when: ssh_key_rotation_sshd_effective_config.rc == 0 + ansible.builtin.assert: + that: >- + (ssh_key_rotation_key_type_signature_algorithms[ssh_key_rotation_new_key_type] | default([ssh_key_rotation_new_key_type | lower])) + | intersect(ssh_key_rotation_effective_pubkey_algorithms) | length > 0 + fail_msg: >- + sshd -T on this host reports PubkeyAcceptedAlgorithms={{ ssh_key_rotation_effective_pubkey_algorithms | join(',') }}, + which does not include any signature algorithm for a {{ ssh_key_rotation_new_key_type }} key + ({{ ssh_key_rotation_key_type_signature_algorithms[ssh_key_rotation_new_key_type] | default([ssh_key_rotation_new_key_type]) | join(',') }}). + This host's crypto-policy would reject the new key entirely. Generate a key type this host's + PubkeyAcceptedAlgorithms accepts (see README.md's Key Validation section), or adjust + pqc_pubkey_algorithms/manage_crypto_policy, before continuing. Aborting now, before + the verify stage would remove the old key - nothing has been broken. + success_msg: "Target sshd will accept the new {{ ssh_key_rotation_new_key_type }} key's signature algorithm." + + - name: Install | Determine which requested PQC algorithms are missing from the effective sshd config + when: + - ssh_key_rotation_sshd_effective_config.rc == 0 + - (ssh_key_rotation_pqc_kex_algorithms + ssh_key_rotation_pqc_pubkey_algorithms) | length > 0 + ansible.builtin.set_fact: + ssh_key_rotation_missing_pqc_algorithms: >- + {{ (ssh_key_rotation_pqc_kex_algorithms | difference(ssh_key_rotation_effective_kex_algorithms)) + + (ssh_key_rotation_pqc_pubkey_algorithms | difference(ssh_key_rotation_effective_pubkey_algorithms)) }} + + - name: Install | Warn if sshd's effective config doesn't appear to include the requested PQC algorithms + when: + - ssh_key_rotation_missing_pqc_algorithms is defined + - ssh_key_rotation_missing_pqc_algorithms | length > 0 + ansible.builtin.debug: + msg: >- + sshd -T on this host does not list: {{ ssh_key_rotation_missing_pqc_algorithms }}. + The verify stage may fail to negotiate these; review sshd_config/crypto-policy before proceeding. + + rescue: + - name: Install rollback | Re-read the recorded state so it includes this stage's own changes + ansible.builtin.include_tasks: state_read.yml + + # Configuration first: removing this role's own files cannot cost anyone access. + - name: Install rollback | Remove the install drop-in if this run created it + when: + - (ssh_key_rotation_recorded.dropins | default({})).install is defined + - ssh_key_rotation_recorded.dropins.install.created_by_role | default(false) | bool + ansible.builtin.file: + path: "{{ ssh_key_rotation_recorded.dropins.install.path }}" + state: absent + become: true + register: ssh_key_rotation_install_rb_dropin + + - name: Install rollback | Restore an install drop-in that existed before this run + when: + - (ssh_key_rotation_recorded.dropins | default({})).install is defined + - ssh_key_rotation_recorded.dropins.install.backup | default(none) is not none + ansible.builtin.copy: + src: "{{ ssh_key_rotation_recorded.dropins.install.backup }}" + dest: "{{ ssh_key_rotation_recorded.dropins.install.path }}" + remote_src: true + mode: preserve + become: true + register: ssh_key_rotation_install_rb_dropin_restored + + # Pre-8.2 hosts only, where the marked block had to go into the main file. + - name: Install rollback | Remove the marked block from sshd_config on hosts without drop-ins + when: not (ssh_key_rotation_use_dropin | default(true) | bool) + ansible.builtin.blockinfile: + path: /etc/ssh/sshd_config + marker: "# {mark} krameff.ssh_key_rotation install" + state: absent + validate: 'sshd -t -f %s' + become: true + register: ssh_key_rotation_install_rb_block + + # `.dest is defined`, not `is succeeded`: a SKIPPED result also satisfies `is succeeded`, and + # referencing .dest on one raises inside the rescue - which aborts the rescue and cancels the + # rollback entirely. + - name: Install rollback | Prove the old key still works before touching authorized_keys + when: + - (ssh_key_rotation_recorded.authorized_keys | default({})).backup | default(none) is not none + - old_private_key is defined + ansible.builtin.include_tasks: probe_key.yml + vars: + ssh_key_rotation_probe_key: "{{ old_private_key }}" + ssh_key_rotation_probe_label: old key + ssh_key_rotation_probe_required: false + + - name: Install rollback | Restore the pre-install authorized_keys + when: + - (ssh_key_rotation_recorded.authorized_keys | default({})).backup | default(none) is not none + - ssh_key_rotation_probe_result is defined and ssh_key_rotation_probe_result.rc == 0 + ansible.builtin.copy: + src: "{{ ssh_key_rotation_recorded.authorized_keys.backup }}" + dest: "~{{ ssh_key_rotation_target_user }}/.ssh/authorized_keys" + remote_src: true + mode: preserve + owner: "{{ ssh_key_rotation_target_user }}" + group: "{{ ssh_key_rotation_recorded.target_group | default(ssh_key_rotation_target_group) }}" + become: true + register: ssh_key_rotation_install_rb_ak + + # Only if a configuration file actually changed back. authorized_keys needs no reload at all, + # since sshd reads it per new connection. + - name: Install rollback | Reload sshd only if its configuration was changed back + when: >- + (ssh_key_rotation_install_rb_dropin is defined and ssh_key_rotation_install_rb_dropin is changed) + or (ssh_key_rotation_install_rb_dropin_restored is defined and ssh_key_rotation_install_rb_dropin_restored is changed) + or (ssh_key_rotation_install_rb_block is defined and ssh_key_rotation_install_rb_block is changed) + ansible.builtin.service: + name: "{{ ssh_key_rotation_sshd_service }}" + state: reloaded + become: true + register: ssh_key_rotation_install_rb_reload + ignore_errors: true + + - name: Install rollback | Record the rollback in the host's rotation state + ansible.builtin.include_tasks: state_write.yml + vars: + ssh_key_rotation_state: + stage: install-rolled-back + + # Guarded: an undefined variable referenced inside a rescue aborts the rescue and cancels the + # rollback, so this never assumes old_private_key was supplied. + - name: Install rollback | Prove the old key still authenticates after the rollback + when: old_private_key is defined + ansible.builtin.include_tasks: probe_key.yml + vars: + ssh_key_rotation_probe_key: "{{ old_private_key }}" + ssh_key_rotation_probe_label: old key + ssh_key_rotation_probe_required: false + + - name: Install rollback | Fail with details of what was automatically undone + ansible.builtin.fail: + msg: >- + Install stage failed and was automatically rolled back. + Original error: {{ ansible_failed_result.msg | default(ansible_failed_result) }}. + {{ ('CRITICAL: the old key could NOT authenticate on a fresh connection after the rollback' + ~ ' (ssh exited ' ~ (ssh_key_rotation_probe_result.rc | default('unknown') | string) + ~ ': ' ~ (ssh_key_rotation_probe_result.stderr | default('') | trim) + ~ '). Do not close existing sessions; recover via console if needed.') + if (ssh_key_rotation_probe_result is not defined or ssh_key_rotation_probe_result.rc != 0) + else 'The old key was re-proven on a fresh, unmultiplexed connection after the rollback.' }} + {{ ("This role's sshd configuration was removed.") + if (ssh_key_rotation_use_dropin | default(true) | bool) + else ("This role's marked block was removed from sshd_config.") }} + {{ 'authorized_keys was restored to its pre-install contents, so the new key is no longer installed.' + if (ssh_key_rotation_install_rb_ak is defined and ssh_key_rotation_install_rb_ak is changed) + else 'authorized_keys was left as-is: no pre-install backup was recorded, or the old key could not be proven to work.' }} + {{ ('WARNING: sshd could not be reloaded during the rollback (' + ~ (ssh_key_rotation_install_rb_reload.msg | default('unknown error')) + ~ ') - the restored configuration is on disk but is NOT yet active.') + if (ssh_key_rotation_install_rb_reload is defined and ssh_key_rotation_install_rb_reload is failed) + else '' }} diff --git a/roles/ssh_key_rotation/tasks/resolve_sshd_anchor.yml b/roles/ssh_key_rotation/tasks/resolve_sshd_anchor.yml new file mode 100644 index 0000000..103aed0 --- /dev/null +++ b/roles/ssh_key_rotation/tasks/resolve_sshd_anchor.yml @@ -0,0 +1,43 @@ +--- + +# Shared by install.yml and verify.yml. Works out WHERE in /etc/ssh/sshd_config this role's +# managed block belongs, so that every directive it writes lands in the GLOBAL section. +# +# This exists because of a real bug. Both stages used to write settings with `lineinfile` and a +# regexp matching the keyword anywhere in the file. `lineinfile` replaces the LAST match, and +# `Match` blocks live at the END of sshd_config, so on any host with a +# `Match ... PasswordAuthentication` block the edit landed INSIDE that block: it changed the +# policy for the matched users and left the global setting untouched, which is the opposite of +# the intent in both directions. Confirmed on Rocky 9.8. +# +# Placement rules, in order: +# 1. After the `Include` line, when there is one. sshd keeps the FIRST value it sees for a +# keyword and the Include sits at the top, so this deliberately leaves existing drop-ins +# outranking us - their precedence is preserved, and the verify stage fails loudly if one +# of them defeats the lock-down rather than silently working around it. +# 2. Otherwise before the first `Match`, so the block can never land inside a conditional. +# 3. Otherwise end of file, which is still the global section when no Match block exists. +# +# Being early also means the block wins over any later global occurrence of the same keyword, so +# stale duplicates further down the file do not need rewriting. + +- name: Resolve sshd anchor | Find sshd_config's Include and Match directives + ansible.builtin.command: "grep -nE '^[[:space:]]*(Include|Match)[[:space:]]' /etc/ssh/sshd_config" + become: true + changed_when: false + failed_when: false + register: ssh_key_rotation_sshd_anchors + +# Plain strings, empty when not applicable, rather than storing `omit` in a fact: the omit +# placeholder is resolved when a task's arguments are templated, and round-tripping it through +# set_fact is fragile. Callers turn "" back into omit with `| default(omit, true)`. +- name: Resolve sshd anchor | Decide where the managed block belongs + vars: + ssh_key_rotation_has_include: "{{ ssh_key_rotation_sshd_anchors.stdout is search('(?im)^\\d+:\\s*Include\\s') }}" + ssh_key_rotation_has_match: "{{ ssh_key_rotation_sshd_anchors.stdout is search('(?im)^\\d+:\\s*Match\\s') }}" + ansible.builtin.set_fact: + ssh_key_rotation_block_after: "{{ '^[ \\t]*Include[ \\t]' if ssh_key_rotation_has_include | bool else '' }}" + ssh_key_rotation_block_before: >- + {{ '^[ \t]*Match[ \t]' + if (not (ssh_key_rotation_has_include | bool)) and (ssh_key_rotation_has_match | bool) + else '' }} diff --git a/roles/ssh_key_rotation/tasks/resolve_target_identity.yml b/roles/ssh_key_rotation/tasks/resolve_target_identity.yml new file mode 100644 index 0000000..222e712 --- /dev/null +++ b/roles/ssh_key_rotation/tasks/resolve_target_identity.yml @@ -0,0 +1,30 @@ +--- + +# Shared by install.yml and verify.yml. Both stages copy authorized_keys around while running +# with become, so they need to know who the file must end up owned by. The primary group is +# looked up rather than assumed equal to the username: that holds on Debian/Ubuntu USERGROUPS +# systems, but not everywhere (e.g. a user whose primary group is `users` or `staff`), and +# passing a non-existent group name makes the copy fail outright. + +# fail_key: false so a missing user leaves getent_passwd empty instead of failing the task with +# the module's generic "One or more supplied key could not be found in the database". Without it +# the task below never runs and the operator never sees which variable is actually wrong. +- name: Resolve target identity | Look up the target user in passwd + ansible.builtin.getent: + database: passwd + key: "{{ ssh_key_rotation_target_user }}" + fail_key: false + +- name: Resolve target identity | Fail clearly if the target user does not exist on this host + ansible.builtin.assert: + that: ssh_key_rotation_target_user in ansible_facts.getent_passwd + fail_msg: >- + ssh_key_rotation_target_user ({{ ssh_key_rotation_target_user }}) does not exist on this + host, so its authorized_keys cannot be managed. Check ansible_user / the inventory. + +# getent_passwd fields are [password, uid, gid, gecos, dir, shell]; index 2 is the numeric GID. +# A numeric GID is used directly, so no name lookup is needed and the value can never be a +# group name that does not resolve. +- name: Resolve target identity | Record the target user's primary GID + ansible.builtin.set_fact: + ssh_key_rotation_target_group: "{{ ansible_facts.getent_passwd[ssh_key_rotation_target_user][2] }}" diff --git a/roles/ssh_key_rotation/tasks/verify.yml b/roles/ssh_key_rotation/tasks/verify.yml index 17afce2..be49fae 100644 --- a/roles/ssh_key_rotation/tasks/verify.yml +++ b/roles/ssh_key_rotation/tasks/verify.yml @@ -6,6 +6,28 @@ - name: Verify | New KEY | Gather facts ansible.builtin.setup: +- name: Verify | Resolve the target user's primary group + ansible.builtin.include_tasks: resolve_target_identity.yml + +- name: Verify | Decide how this host's sshd will be configured + ansible.builtin.include_tasks: resolve_sshd_strategy.yml + +# Read from the host, not carried over from the install play. This is what lets the rollback below +# undo the INSTALL stage's changes too, and what keeps the verify stage usable standalone. Anything +# missing or malformed is treated as "no install state", which degrades to rolling back only this +# stage's changes rather than erroring inside a rescue. +- name: Verify | Read the rotation state the install stage recorded on this host + ansible.builtin.include_tasks: state_read.yml + +- name: Verify | Warn when no install-stage state was found + when: ssh_key_rotation_recorded | length == 0 + ansible.builtin.debug: + msg: >- + No rotation state found at /etc/ansible/facts.d/ssh_key_rotation.fact on this host. If the + verify stage fails, its rollback will undo only the changes this stage made - install-stage + changes (the new key in authorized_keys, and this role's install drop-in) will be left in + place. That is expected when the verify stage is run on its own. + # Hard safety gate: if the new key can't authenticate, this fails the host and nothing below (the destructive part) executes. - name: Verify | New KEY | Confirm the host is reachable ansible.builtin.ping: @@ -25,9 +47,11 @@ - name: Verify | New KEY | Back up authorized_keys before removing the old key ansible.builtin.copy: src: "~{{ ssh_key_rotation_target_user }}/.ssh/authorized_keys" - dest: "~{{ ssh_key_rotation_target_user }}/.ssh/authorized_keys.bak-{{ ansible_date_time.iso8601_basic_short }}" + dest: "~{{ ssh_key_rotation_target_user }}/.ssh/authorized_keys.bak-{{ ansible_facts.date_time.iso8601_basic_short }}" remote_src: true mode: preserve + owner: "{{ ssh_key_rotation_target_user }}" + group: "{{ ssh_key_rotation_target_group }}" register: ssh_key_rotation_authorized_keys_backup - name: Verify | New KEY | Remove the OLD public key from authorized_keys @@ -44,35 +68,64 @@ state: present exclusive: true - # Explicit, deterministically-named backup (same pattern as authorized_keys above), so rescue: below - # knows exactly which file to restore, rather than chasing lineinfile's own auto-named backup files. - - name: Verify | Back up sshd_config before disabling legacy auth - when: (ssh_key_rotation_disable_password_auth | bool) or (ssh_key_rotation_disable_kbd_interactive | bool) + # Written to its own drop-in, separate from the install stage's file. Two files rather than one + # on purpose: this stage's rescue deletes the lock-down file, and on a re-run after a previously + # successful install that would otherwise also delete PQC algorithms an earlier run legitimately + # established - which, for a client that can only negotiate those algorithms, is itself a + # lockout. Separate files also cannot collide on the same keyword. + - name: Verify | Write the lock-down settings to a drop-in + when: + - ssh_key_rotation_use_dropin | bool + - (ssh_key_rotation_disable_password_auth | bool) or (ssh_key_rotation_disable_kbd_interactive | bool) + ansible.builtin.include_tasks: sshd_dropin_write.yml + vars: + ssh_key_rotation_dropin_name: ssh-key-rotation-lockdown.conf + ssh_key_rotation_dropin_key: lockdown + ssh_key_rotation_dropin_content: | + # Managed by krameff.ssh_key_rotation (verify stage). Safe to remove; this role rewrites it. + {% if ssh_key_rotation_disable_password_auth | bool %} + PasswordAuthentication no + {% endif %} + {% if ssh_key_rotation_disable_kbd_interactive | bool %} + KbdInteractiveAuthentication no + {% endif %} + + # Pre-OpenSSH-8.2 fallback only: no Include support, so the main file must be edited. Backed up + # first, and anchored to the global section so the edit can never land inside a Match block. + - name: Verify | Back up sshd_config before editing it on a host without drop-in support + when: + - not (ssh_key_rotation_use_dropin | bool) + - (ssh_key_rotation_disable_password_auth | bool) or (ssh_key_rotation_disable_kbd_interactive | bool) ansible.builtin.copy: src: /etc/ssh/sshd_config - dest: "/etc/ssh/sshd_config.bak-{{ ansible_date_time.iso8601_basic_short }}" + dest: "/etc/ssh/sshd_config.bak-{{ ansible_facts.date_time.iso8601_basic_short }}" remote_src: true mode: preserve become: true register: ssh_key_rotation_sshd_config_backup - - name: Verify | Disable password authentication (the old way of connecting) - when: ssh_key_rotation_disable_password_auth | bool - ansible.builtin.lineinfile: - path: /etc/ssh/sshd_config - regexp: '^#?\s*PasswordAuthentication' - line: 'PasswordAuthentication no' - validate: 'sshd -t -f %s' - backup: true - become: true - notify: Reload sshd + - name: Verify | Work out where the managed lock-down block belongs + when: + - not (ssh_key_rotation_use_dropin | bool) + - (ssh_key_rotation_disable_password_auth | bool) or (ssh_key_rotation_disable_kbd_interactive | bool) + ansible.builtin.include_tasks: resolve_sshd_anchor.yml - - name: Verify | Disable keyboard-interactive authentication - when: ssh_key_rotation_disable_kbd_interactive | bool - ansible.builtin.lineinfile: + - name: Verify | Disable password and keyboard-interactive auth in the global section + when: + - not (ssh_key_rotation_use_dropin | bool) + - (ssh_key_rotation_disable_password_auth | bool) or (ssh_key_rotation_disable_kbd_interactive | bool) + ansible.builtin.blockinfile: path: /etc/ssh/sshd_config - regexp: '^#?\s*KbdInteractiveAuthentication' - line: 'KbdInteractiveAuthentication no' + marker: "# {mark} krameff.ssh_key_rotation lock-down" + insertafter: "{{ ssh_key_rotation_block_after | default(omit, true) }}" + insertbefore: "{{ ssh_key_rotation_block_before | default(omit, true) }}" + block: | + {% if ssh_key_rotation_disable_password_auth | bool %} + PasswordAuthentication no + {% endif %} + {% if ssh_key_rotation_disable_kbd_interactive | bool %} + KbdInteractiveAuthentication no + {% endif %} validate: 'sshd -t -f %s' backup: true become: true @@ -81,22 +134,243 @@ - name: Verify | Apply final sshd changes ansible.builtin.meta: flush_handlers + # Asserted rather than assumed: the tasks above only edit the GLOBAL section of the main + # sshd_config, but sshd reads `Include /etc/ssh/sshd_config.d/*.conf` (placed at the TOP of + # the file on Debian/Ubuntu and RHEL alike) and takes the FIRST value it sees for each + # keyword. A drop-in such as cloud-init's 50-cloud-init.conf therefore wins outright, and + # the lineinfile tasks still report "changed" while the setting never takes effect. Without + # this check the role reports a successful lock-down on a host where password login still works. + # failed_when: false, and every assertion below gated on rc == 0, deliberately mirroring the + # same call in install.yml. `sshd -T` is a diagnostic here, not the thing being changed: it + # legitimately exits non-zero on hosts where it cannot load host keys, or where the binary + # isn't on the become shell's PATH. Letting it fail hard at this point - AFTER the old key + # has been removed - would trigger the rollback and undo a rotation that actually succeeded, + # which is strictly worse than not being able to run the check. + - name: Verify | Re-read the effective sshd configuration after lock-down + ansible.builtin.command: sshd -T + become: true + changed_when: false + failed_when: false + register: ssh_key_rotation_sshd_effective_after_lockdown + + - name: Verify | Warn if the effective sshd configuration could not be read back + when: ssh_key_rotation_sshd_effective_after_lockdown.rc != 0 + ansible.builtin.debug: + msg: >- + `sshd -T` exited {{ ssh_key_rotation_sshd_effective_after_lockdown.rc }} + ({{ ssh_key_rotation_sshd_effective_after_lockdown.stderr | default('no stderr') | trim }}), + so this run could NOT confirm that password/keyboard-interactive authentication are + actually disabled. The sshd_config edits were applied and validated, but a drop-in under + /etc/ssh/sshd_config.d/ may still override them - verify manually on this host. + + # Scoped to the global section: `sshd -T` without -C reports global values, so a Match block + # that re-enables password auth for some user still passes this. install.yml already warns + # when Match blocks are present; this check closes the drop-in hole, not the Match one. + - name: Verify | Assert password authentication is actually disabled + when: + - ssh_key_rotation_disable_password_auth | bool + - ssh_key_rotation_sshd_effective_after_lockdown.rc == 0 + ansible.builtin.assert: + that: ssh_key_rotation_sshd_effective_after_lockdown.stdout is search('(?im)^passwordauthentication\s+no$') + fail_msg: >- + sshd -T still reports password authentication as ENABLED after this role disabled it in + /etc/ssh/sshd_config. A drop-in under /etc/ssh/sshd_config.d/ almost certainly overrides + it - sshd honours the FIRST value for a keyword and the Include sits at the top of the + main config. Set PasswordAuthentication no in the winning drop-in (or remove it there), + then re-run. Failing here rather than reporting a lock-down that did not happen. + success_msg: >- + Password authentication is confirmed disabled in sshd's global effective configuration. + + # Accepts either spelling: KbdInteractiveAuthentication only exists from OpenSSH 8.7, and + # older releases still shipped by supported EL versions report the setting under its previous + # name, ChallengeResponseAuthentication. Matching only the new name would fail this assert on + # a correctly locked-down host - after the old key was already removed. + - name: Verify | Assert keyboard-interactive authentication is actually disabled + when: + - ssh_key_rotation_disable_kbd_interactive | bool + - ssh_key_rotation_sshd_effective_after_lockdown.rc == 0 + ansible.builtin.assert: + that: >- + ssh_key_rotation_sshd_effective_after_lockdown.stdout is search('(?im)^kbdinteractiveauthentication\s+no$') + or ssh_key_rotation_sshd_effective_after_lockdown.stdout is search('(?im)^challengeresponseauthentication\s+no$') + fail_msg: >- + sshd -T still reports keyboard-interactive authentication as ENABLED after this role + disabled it (checked as both KbdInteractiveAuthentication and the pre-OpenSSH-8.7 name + ChallengeResponseAuthentication). As above, check for an overriding drop-in under + /etc/ssh/sshd_config.d/. + success_msg: "Keyboard-interactive authentication is confirmed disabled in sshd's effective configuration." + + # The checks above read the GLOBAL config: `sshd -T` without -C does not evaluate Match blocks + # at all. A `Match User ` block re-enabling password auth for exactly the account being + # rotated therefore passes them while that account can still log in with a password. Verified + # on Rocky 9.8: with a global `PasswordAuthentication no` plus `Match User vagrant` setting it + # back to yes, `sshd -T` reports no and `sshd -T -C user=vagrant,...` reports yes. + # + # The connect spec uses the real client address from SSH_CONNECTION where available, so + # `Match Address`/`Match Host` blocks are evaluated against the address this rotation actually + # came from rather than a placeholder. + - name: Verify | Determine the client address for the per-user sshd check + when: ssh_key_rotation_check_match_blocks | bool + ansible.builtin.set_fact: + ssh_key_rotation_client_addr: >- + {{ (ansible_facts.env.SSH_CONNECTION | default('')).split(' ') | first | default('', true) or '0.0.0.0' }} + + # failed_when: false - `sshd -T -C` is a diagnostic, and this runs AFTER the old key is gone. + # Some builds reject the connect spec (or -C outright), and failing hard there would roll back + # a rotation that actually succeeded. An unusable check warns; a check that runs and finds + # password auth still live for this user is a hard failure. + - name: Verify | Re-read the effective sshd configuration for the rotated user specifically + when: ssh_key_rotation_check_match_blocks | bool + ansible.builtin.command: >- + sshd -T -C user={{ ssh_key_rotation_target_user | quote }},host=localhost,addr={{ ssh_key_rotation_client_addr | quote }} + become: true + changed_when: false + failed_when: false + register: ssh_key_rotation_sshd_effective_for_user + + - name: Verify | Warn if the per-user sshd configuration could not be read + when: + - ssh_key_rotation_check_match_blocks | bool + - ssh_key_rotation_sshd_effective_for_user.rc != 0 + ansible.builtin.debug: + msg: >- + `sshd -T -C` exited {{ ssh_key_rotation_sshd_effective_for_user.rc }} + ({{ ssh_key_rotation_sshd_effective_for_user.stderr | default('no stderr') | trim }}), + so Match blocks could NOT be evaluated for {{ ssh_key_rotation_target_user }}. The global + configuration was verified, but a `Match User {{ ssh_key_rotation_target_user }}` block + could still re-enable password login for this account. Check by hand with: + sshd -T -C user={{ ssh_key_rotation_target_user }},host=localhost,addr=1.2.3.4 + + - name: Verify | Assert no Match block re-enables password auth for the rotated user + when: + - ssh_key_rotation_check_match_blocks | bool + - ssh_key_rotation_disable_password_auth | bool + - ssh_key_rotation_sshd_effective_for_user.rc == 0 + ansible.builtin.assert: + that: ssh_key_rotation_sshd_effective_for_user.stdout is search('(?im)^passwordauthentication\s+no$') + fail_msg: >- + Password authentication is disabled globally but STILL ENABLED for + {{ ssh_key_rotation_target_user }}, so a Match block in sshd_config re-enables it for + exactly the account just rotated. This role only edits the global section and never + touches Match blocks. Remove or amend the block, then re-run. Find it with: + grep -n Match /etc/ssh/sshd_config /etc/ssh/sshd_config.d/*.conf + success_msg: >- + Password authentication is confirmed disabled for {{ ssh_key_rotation_target_user }} + specifically, so no Match block re-enables it. + + - name: Verify | Assert no Match block re-enables keyboard-interactive auth for the rotated user + when: + - ssh_key_rotation_check_match_blocks | bool + - ssh_key_rotation_disable_kbd_interactive | bool + - ssh_key_rotation_sshd_effective_for_user.rc == 0 + ansible.builtin.assert: + that: >- + ssh_key_rotation_sshd_effective_for_user.stdout is search('(?im)^kbdinteractiveauthentication\s+no$') + or ssh_key_rotation_sshd_effective_for_user.stdout is search('(?im)^challengeresponseauthentication\s+no$') + fail_msg: >- + Keyboard-interactive authentication is disabled globally but STILL ENABLED for + {{ ssh_key_rotation_target_user }}, so a Match block re-enables it for that account. + Remove or amend the block, then re-run. + success_msg: >- + Keyboard-interactive authentication is confirmed disabled for + {{ ssh_key_rotation_target_user }} specifically. + + # authorized_keys must be owned by the target user: sshd's StrictModes (enabled by default) + # silently IGNORES the file otherwise, which would reject every key at once. + - name: Verify | Confirm authorized_keys ownership still satisfies sshd StrictModes + ansible.builtin.stat: + path: "~{{ ssh_key_rotation_target_user }}/.ssh/authorized_keys" + register: ssh_key_rotation_authorized_keys_final_stat + + - name: Verify | Assert authorized_keys is owned by the target user and not group/world writable + ansible.builtin.assert: + that: + - ssh_key_rotation_authorized_keys_final_stat.stat.exists + - ssh_key_rotation_authorized_keys_final_stat.stat.pw_name == ssh_key_rotation_target_user + - not ssh_key_rotation_authorized_keys_final_stat.stat.wgrp + - not ssh_key_rotation_authorized_keys_final_stat.stat.woth + fail_msg: >- + ~{{ ssh_key_rotation_target_user }}/.ssh/authorized_keys is owned by + {{ ssh_key_rotation_authorized_keys_final_stat.stat.pw_name | default('(missing file)') }} + with mode {{ ssh_key_rotation_authorized_keys_final_stat.stat.mode | default('n/a') }}. + sshd's StrictModes will ignore this file entirely, rejecting every key for this user. + success_msg: "authorized_keys ownership and permissions satisfy sshd StrictModes." + + # The connection opened earlier in this play is multiplexed (ControlPersist), so a plain ping + # here would succeed over the EXISTING socket and prove nothing about the lock-down just + # applied. Dropping the connection first makes the check below a real re-authentication. + - name: Verify | Drop the multiplexed connection so the final check re-authenticates + ansible.builtin.meta: reset_connection + - name: Verify | Final connectivity check with the NEW key after lock-down ansible.builtin.ping: rescue: - # Guarded with `is succeeded`, not just `is defined`: if the very first task above (the backup itself) - # is what failed, nothing has actually changed yet, so there's nothing to restore. - - name: Rollback | Restore authorized_keys from the pre-cleanup backup - when: ssh_key_rotation_authorized_keys_backup is succeeded + # Order here is deliberate and safety-critical. + # + # Configuration is undone first, because that cannot cost anyone access: removing this role's + # drop-ins can only restore the host's previous behaviour. Key material is touched last, and + # only after proving the credential we would fall back to actually works. + # + # Everything is driven from the state the install stage recorded on the HOST, not from + # variables registered in an earlier play. An undefined variable inside a rescue: aborts the + # rescue, which would leave a half-changed host with no rollback at all - strictly worse than + # having none. Hence `| default(...)` on every lookup. + + # Re-read first. The snapshot taken at the start of this stage predates the lock-down drop-in + # being recorded, so rolling back from it would leave that file behind - undoing everything + # except the setting this stage exists to apply. + - name: Rollback | Re-read the recorded state so it includes this stage's own changes + ansible.builtin.include_tasks: state_read.yml + + - name: Rollback | Remove this stage's lock-down drop-in + when: (ssh_key_rotation_recorded.dropins | default({})).lockdown is defined + ansible.builtin.file: + path: "{{ ssh_key_rotation_recorded.dropins.lockdown.path }}" + state: absent + become: true + register: ssh_key_rotation_rollback_lockdown_removed + + - name: Rollback | Restore a lock-down drop-in that existed before this run + when: + - (ssh_key_rotation_recorded.dropins | default({})).lockdown is defined + - ssh_key_rotation_recorded.dropins.lockdown.backup | default(none) is not none ansible.builtin.copy: - src: "{{ ssh_key_rotation_authorized_keys_backup.dest }}" - dest: "~{{ ssh_key_rotation_target_user }}/.ssh/authorized_keys" + src: "{{ ssh_key_rotation_recorded.dropins.lockdown.backup }}" + dest: "{{ ssh_key_rotation_recorded.dropins.lockdown.path }}" + remote_src: true + mode: preserve + become: true + + # Only when THIS run created it. A drop-in that already existed belongs to the administrator, + # and deleting it would destroy configuration this role never owned. + - name: Rollback | Remove the install stage's drop-in if this run created it + when: + - (ssh_key_rotation_recorded.dropins | default({})).install is defined + - ssh_key_rotation_recorded.dropins.install.created_by_role | default(false) | bool + ansible.builtin.file: + path: "{{ ssh_key_rotation_recorded.dropins.install.path }}" + state: absent + become: true + register: ssh_key_rotation_rollback_install_removed + + - name: Rollback | Restore an install drop-in that existed before this run + when: + - (ssh_key_rotation_recorded.dropins | default({})).install is defined + - ssh_key_rotation_recorded.dropins.install.backup | default(none) is not none + ansible.builtin.copy: + src: "{{ ssh_key_rotation_recorded.dropins.install.backup }}" + dest: "{{ ssh_key_rotation_recorded.dropins.install.path }}" remote_src: true mode: preserve + become: true - - name: Rollback | Restore sshd_config from the pre-cleanup backup - when: ssh_key_rotation_sshd_config_backup is defined and ssh_key_rotation_sshd_config_backup is succeeded + # Pre-8.2 hosts only, where the main file had to be edited rather than a drop-in written. + - name: Rollback | Restore sshd_config on a host without drop-in support + # `.dest is defined`, not `is succeeded`: a SKIPPED task result also satisfies `is succeeded`, + # so the old guard passed on drop-in hosts where this backup never ran, and referencing .dest + # raised inside the rescue - aborting the rescue and cancelling the whole rollback. + when: ssh_key_rotation_sshd_config_backup.dest | default(none) is not none ansible.builtin.copy: src: "{{ ssh_key_rotation_sshd_config_backup.dest }}" dest: /etc/ssh/sshd_config @@ -104,15 +378,15 @@ mode: preserve become: true - # A failed reload must not abort the rollback. authorized_keys and sshd_config are already - # restored on disk above, and sshd reads authorized_keys per-connection, so access is back even - # if the daemon never re-read its config. rescue: has no rescue of its own, so letting this fail - # would skip the two tasks that matter most here: the connectivity check below (the real access - # gate) and the diagnostic that tells the operator what was restored. Reported in that message - # instead. Note the reload can fail for the very same reason the block did - both use - # ssh_key_rotation_sshd_service - so this is the expected path, not an edge case. - - name: Rollback | Reload sshd with the restored configuration - when: ssh_key_rotation_sshd_config_backup is defined and ssh_key_rotation_sshd_config_backup is succeeded + # Only if a configuration file actually changed. authorized_keys needs no reload at all - sshd + # reads it per new connection - so a reload here is about config only. Skipping it when nothing + # changed also avoids the old wart where the rollback's reload failed for the very same reason + # the block did. + - name: Rollback | Reload sshd only if its configuration was actually changed back + when: >- + (ssh_key_rotation_rollback_lockdown_removed is defined and ssh_key_rotation_rollback_lockdown_removed is changed) + or (ssh_key_rotation_rollback_install_removed is defined and ssh_key_rotation_rollback_install_removed is changed) + or (ssh_key_rotation_sshd_config_backup.dest | default(none) is not none) ansible.builtin.service: name: "{{ ssh_key_rotation_sshd_service }}" state: reloaded @@ -120,25 +394,164 @@ register: ssh_key_rotation_rollback_reload ignore_errors: true - # Stays fatal on purpose: this is what proves access survived the rollback. - - name: Rollback | Confirm connectivity still works after rollback - ansible.builtin.ping: + # The old key is proven to work BEFORE anything is removed from authorized_keys. Restoring the + # pre-install file deletes the new key, and if the old key then turns out not to work - a + # previous run used make_exclusive, the key is passphrase-protected, or a crypto policy no + # longer accepts its algorithm - that would delete the last usable credential from a host we + # are still connected to, with the multiplexed session hiding it until it drops. + # Restoring the pre-install file is what puts the OLD key back: this stage removed it before + # failing, and leaving it removed is not "as we found it". + # + # Deliberately NOT gated on probing the old key first. That ordering is circular - the old key + # has already been removed by this point, so a probe of it cannot succeed, and gating on it + # meant the restore never ran. The restore is safe without it: the new key is put straight back + # alongside the old one below, so the host is never left with fewer credentials than it had a + # moment earlier. sshd reads authorized_keys per new connection, so an established session is + # unaffected either way. + - name: Rollback | Restore the pre-install authorized_keys + when: (ssh_key_rotation_recorded.authorized_keys | default({})).backup | default(none) is not none + ansible.builtin.copy: + src: "{{ ssh_key_rotation_recorded.authorized_keys.backup }}" + dest: "~{{ ssh_key_rotation_target_user }}/.ssh/authorized_keys" + remote_src: true + mode: preserve + owner: "{{ ssh_key_rotation_target_user }}" + group: "{{ ssh_key_rotation_recorded.target_group | default(ssh_key_rotation_target_group) }}" + become: true + register: ssh_key_rotation_rollback_ak_restored + + # Default: old key back AND new key kept. Removing the new key would leave the host depending + # solely on the old key working, which is not guaranteed - a previous run may have used + # make_exclusive, the key may be passphrase-protected, or a crypto policy may no longer accept + # its algorithm. Keeping both means the rollback cannot cost anyone access. + - name: Rollback | Put the new key back alongside the restored old key + when: + - not (ssh_key_rotation_rollback_remove_new_key | bool) + - ssh_key_rotation_rollback_ak_restored is defined and ssh_key_rotation_rollback_ak_restored is changed + ansible.posix.authorized_key: + user: "{{ ssh_key_rotation_target_user }}" + key: "{{ lookup('file', new_public_key_file) }}" + state: present + exclusive: false + become: true + + # Safety net for the exact-restore mode: if the old key turns out not to work, the host would + # now have no usable credential at all, so the new key goes back regardless of the setting. + - name: Rollback | Check the old key works when the new key was deliberately removed + when: + - ssh_key_rotation_rollback_remove_new_key | bool + - ssh_key_rotation_rollback_ak_restored is defined and ssh_key_rotation_rollback_ak_restored is changed + - old_private_key is defined + ansible.builtin.include_tasks: probe_key.yml + vars: + ssh_key_rotation_probe_key: "{{ old_private_key }}" + ssh_key_rotation_probe_label: old key + ssh_key_rotation_probe_required: false + + - name: Rollback | Re-install the new key because the old key could not authenticate + when: + - ssh_key_rotation_rollback_remove_new_key | bool + - ssh_key_rotation_rollback_ak_restored is defined and ssh_key_rotation_rollback_ak_restored is changed + - ssh_key_rotation_probe_result is defined and ssh_key_rotation_probe_result.rc != 0 + ansible.posix.authorized_key: + user: "{{ ssh_key_rotation_target_user }}" + key: "{{ lookup('file', new_public_key_file) }}" + state: present + exclusive: false + become: true + register: ssh_key_rotation_rollback_new_key_readded + + # StrictModes gate: sshd silently IGNORES an authorized_keys not owned by the target user, + # refusing every key for that account. A restore that gets ownership wrong is itself a lockout, + # so this is checked rather than assumed. + - name: Rollback | Confirm authorized_keys ownership after any restore + ansible.builtin.stat: + path: "~{{ ssh_key_rotation_target_user }}/.ssh/authorized_keys" + register: ssh_key_rotation_rollback_ak_stat + + - name: Rollback | Warn if authorized_keys ownership would defeat sshd StrictModes + when: >- + (not ssh_key_rotation_rollback_ak_stat.stat.exists) + or ssh_key_rotation_rollback_ak_stat.stat.pw_name != ssh_key_rotation_target_user + or ssh_key_rotation_rollback_ak_stat.stat.wgrp or ssh_key_rotation_rollback_ak_stat.stat.woth + ansible.builtin.debug: + msg: >- + WARNING: ~{{ ssh_key_rotation_target_user }}/.ssh/authorized_keys is owned by + {{ ssh_key_rotation_rollback_ak_stat.stat.pw_name | default('(missing file)') }} with mode + {{ ssh_key_rotation_rollback_ak_stat.stat.mode | default('n/a') }}. sshd StrictModes will + ignore it and refuse every key for this user. Fix ownership before closing your session. + + # The final proof, and deliberately NOT ansible.builtin.ping. Ansible's ControlPath is keyed on + # host/port/user, not on the identity file, so a ping can succeed over a socket opened with a + # different credential and report access that no longer exists. probe_key.yml drives a real ssh + # client with ControlMaster=no and ControlPath=none, so its exit code means what it says. + # Which private key is available depends on how the role was invoked: playbooks/rotate.yml + # supplies both, but a caller driving the stages directly with tasks_from may set neither. + # Referencing an undefined one here would raise INSIDE the rescue, which aborts the rescue and + # cancels the rollback - the exact failure this whole design exists to avoid. So the key is + # chosen from what is actually defined, and the proof is skipped with a warning if neither is. + - name: Rollback | Choose a private key to prove access with + ansible.builtin.set_fact: + ssh_key_rotation_rollback_probe_key: >- + {{ old_private_key + if (old_private_key is defined + and ssh_key_rotation_rollback_ak_restored is defined + and ssh_key_rotation_rollback_ak_restored is changed) + else (new_private_key if new_private_key is defined + else (old_private_key if old_private_key is defined else '')) }} + + - name: Rollback | Prove a usable credential still authenticates after the rollback + when: ssh_key_rotation_rollback_probe_key | length > 0 + ansible.builtin.include_tasks: probe_key.yml + vars: + ssh_key_rotation_probe_key: "{{ ssh_key_rotation_rollback_probe_key }}" + ssh_key_rotation_probe_label: restored credential + ssh_key_rotation_probe_required: false + + - name: Rollback | Warn when no private key was available to prove access with + when: ssh_key_rotation_rollback_probe_key | length == 0 + ansible.builtin.debug: + msg: >- + Neither old_private_key nor new_private_key is defined, so access could NOT be re-proven + after the rollback. The configuration changes were undone, but verify by hand that you can + still log in to this host before closing your session. + + - name: Rollback | Record the rollback in the host's rotation state + ansible.builtin.include_tasks: state_write.yml + vars: + ssh_key_rotation_state: + stage: verify-rolled-back - - name: Rollback | Fail with details of what was automatically restored + - name: Rollback | Fail with details of what was automatically undone ansible.builtin.fail: msg: >- Verify stage failed partway through cleanup and was automatically rolled back. Original error: {{ ansible_failed_result.msg | default(ansible_failed_result) }}. - Restored authorized_keys from - {{ ssh_key_rotation_authorized_keys_backup.dest | default('n/a - nothing had changed yet') }} - {{ ('and sshd_config from ' + ssh_key_rotation_sshd_config_backup.dest) - if (ssh_key_rotation_sshd_config_backup is defined and ssh_key_rotation_sshd_config_backup is succeeded) - else '(sshd_config was not touched)' }}. - The old key and prior sshd_config are back in place; nothing was left half-changed. + {{ ('CRITICAL: after rolling back, NO credential could authenticate on a fresh connection' + ~ ' (ssh exited ' ~ (ssh_key_rotation_probe_result.rc | default('unknown') | string) + ~ ': ' ~ (ssh_key_rotation_probe_result.stderr | default('') | trim) + ~ '). Do not close existing sessions; recover via console if needed.') + if (ssh_key_rotation_rollback_probe_key | default('') | length > 0 + and (ssh_key_rotation_probe_result is not defined or ssh_key_rotation_probe_result.rc != 0)) + else ('Access was re-proven on a fresh, unmultiplexed connection after the rollback.' + if (ssh_key_rotation_rollback_probe_key | default('') | length > 0) + else 'No private key was available to re-prove access; verify by hand.') }} + {{ ("This role's drop-ins were removed, so its configuration changes are fully undone.") + if (ssh_key_rotation_recorded.dropins is defined) else + ('sshd_config was restored from ' ~ (ssh_key_rotation_sshd_config_backup.dest | default('n/a', true))) }} + {{ ('authorized_keys was restored, so the old key works again; the new key was deliberately' + ~ ' left alongside it, since removing it would leave the host depending solely on the' + ~ ' old key. Set ssh_key_rotation_rollback_remove_new_key=true for the exact pre-install file.') + if (ssh_key_rotation_rollback_ak_restored is defined + and ssh_key_rotation_rollback_ak_restored is changed + and not (ssh_key_rotation_rollback_remove_new_key | bool)) + else ('authorized_keys was restored to its exact pre-install contents; the new key is no longer installed.' + if (ssh_key_rotation_rollback_ak_restored is defined and ssh_key_rotation_rollback_ak_restored is changed) + else 'authorized_keys was NOT restored - no pre-install backup was recorded, or the old key could not be proven to work - so the new key was left in place as the only usable credential.') }} {{ ('WARNING: sshd could not be reloaded during the rollback (' ~ (ssh_key_rotation_rollback_reload.msg | default('unknown error')) - ~ ') - the restored sshd_config is on disk but is NOT yet active. Reload or restart' - ~ ' sshd manually to activate it. Key-based access is unaffected, since sshd reads' - ~ ' authorized_keys per connection.') + ~ ') - the restored configuration is on disk but is NOT yet active. Reload or restart' + ~ ' sshd manually. Key-based access is unaffected, since sshd reads authorized_keys' + ~ ' per connection.') if (ssh_key_rotation_rollback_reload is defined and ssh_key_rotation_rollback_reload is failed) else '' }} From 627afa604c58616b395c419a0101178e0e3d1cd2 Mon Sep 17 00:00:00 2001 From: Mark Bolwell Date: Thu, 6 Aug 2026 19:50:55 +0100 Subject: [PATCH 13/18] The role writes drop-ins and no longer edits sshd_config, so undo is a deletion. Records what it changed per host in facts.d; both stages roll back their own changes and prove access with a real unmultiplexed ssh. Signed-off-by: Mark Bolwell --- roles/ssh_key_rotation/defaults/main.yml | 48 ++++++++++ roles/ssh_key_rotation/tasks/probe_key.yml | 78 ++++++++++++++++ .../tasks/resolve_sshd_strategy.yml | 34 +++++++ .../tasks/sshd_dropin_write.yml | 92 +++++++++++++++++++ roles/ssh_key_rotation/tasks/state_read.yml | 29 ++++++ roles/ssh_key_rotation/tasks/state_write.yml | 66 +++++++++++++ 6 files changed, 347 insertions(+) create mode 100644 roles/ssh_key_rotation/tasks/probe_key.yml create mode 100644 roles/ssh_key_rotation/tasks/resolve_sshd_strategy.yml create mode 100644 roles/ssh_key_rotation/tasks/sshd_dropin_write.yml create mode 100644 roles/ssh_key_rotation/tasks/state_read.yml create mode 100644 roles/ssh_key_rotation/tasks/state_write.yml diff --git a/roles/ssh_key_rotation/defaults/main.yml b/roles/ssh_key_rotation/defaults/main.yml index ccd039c..fb4347d 100644 --- a/roles/ssh_key_rotation/defaults/main.yml +++ b/roles/ssh_key_rotation/defaults/main.yml @@ -1,5 +1,12 @@ --- +# The four required variables - old_private_key, old_public_key_file, new_private_key and +# new_public_key_file - are deliberately absent from this file. They have no safe default: any +# value here would count as "defined" and permanently disable the validate stage's check that +# catches a missing or misspelled path before a single host is touched. Set them per run, either +# with -e or by copying rotation_vars.example.yml and passing -e @rotation_vars.yml. See +# README.md's Variables section for the full list. + # Default shell to be used for shell commands # Could be overridden by inventory etc ssh_key_rotation_os_default_shell: "/bin/bash" @@ -43,6 +50,47 @@ ssh_key_rotation_disable_password_auth: true # disable password login = "the o ssh_key_rotation_disable_kbd_interactive: true ssh_key_rotation_make_exclusive: false # set true to leave ONLY the new key in authorized_keys +# Where this role writes its own configuration. On any host with an `Include` line (OpenSSH 8.2+, +# which is every currently supported distribution) the role writes drop-ins here and NEVER edits +# /etc/ssh/sshd_config, so rolling back is deleting a file rather than restoring someone else's +# config. Hosts with no Include fall back to a marked block in the main file. +ssh_key_rotation_sshd_dropin_dir: /etc/ssh/sshd_config.d + +# The numeric prefix is load-bearing, and lower means stronger. sshd keeps the FIRST value it sees +# for a keyword and the Include sits at the TOP of sshd_config, so ordering runs: earlier drop-in > +# later drop-in > main file body. Verified on Ubuntu 26.04 (99-* loses to 50-cloud-init.conf, 01-* +# beats it) and Rocky 9.8 (99-* still beats the main file body). +# +# 99 is the deliberate default: it overrides the main file, but an administrator's own drop-in +# still wins. That keeps their intent authoritative and leaves the verify stage's lock-down check +# able to detect the conflict and fail loudly. Set 01 to override admin drop-ins instead - see the +# warning on ssh_key_rotation_manage_sshd_dropin below before you do. +ssh_key_rotation_sshd_dropin_prefix: "99" + +# DEPRECATED, honoured for compatibility: equivalent to setting the prefix to "01". Two things to +# understand before enabling it. First, it makes this role outrank administrator drop-ins, which +# also makes the "lock-down did not take effect" check a tautology - the role's file always wins, +# so a conflicting drop-in can no longer be detected. Second, it is why this role does NOT manage +# AuthorizedKeysFile: at this precedence, overriding a central key store such as +# `AuthorizedKeysFile /etc/ssh/authorized_keys/%u` would strip key access from every OTHER user on +# the host while the rotated user kept working, so the run would report success having locked out +# the rest of the estate. +ssh_key_rotation_manage_sshd_dropin: false + +# Rollback restores the host's configuration, but by default LEAVES the new key in authorized_keys +# rather than restoring the pre-install file. That is a deliberate deviation from "pristine": the +# operator already holds that key so it is no third-party exposure, and removing it means relying +# on the old key still working - which is not guaranteed if a previous run used make_exclusive, or +# the key is passphrase-protected, or a crypto policy no longer accepts its algorithm. Set true to +# restore the pre-install authorized_keys exactly; the rollback then proves the old key +# authenticates BEFORE removing anything, and keeps the new key if that proof fails. +ssh_key_rotation_rollback_remove_new_key: false + +# Checks the lock-down actually applies to the user being rotated, via `sshd -T -C`, not just +# globally: a `Match User` block can re-enable password auth for exactly that account while the +# global config reports it disabled. Set false only if `sshd -T -C` misbehaves on your hosts. +ssh_key_rotation_check_match_blocks: true + # Common to install/verify stages; same user on both ends of the rotation. ssh_key_rotation_target_user: "{{ ansible_user | default(lookup('env', 'USER')) }}" diff --git a/roles/ssh_key_rotation/tasks/probe_key.yml b/roles/ssh_key_rotation/tasks/probe_key.yml new file mode 100644 index 0000000..4d02311 --- /dev/null +++ b/roles/ssh_key_rotation/tasks/probe_key.yml @@ -0,0 +1,78 @@ +--- + +# Proves - or disproves - that a given private key can actually log in to this host, by driving a +# real ssh client from the control node. +# +# Why not ansible.builtin.ping. Ansible's SSH connections are multiplexed, and the default +# ControlPath is keyed on host, port and remote user - NOT on the identity file. So a ping issued +# after switching ansible_ssh_private_key_file can succeed over the socket opened with the PREVIOUS +# key, reporting that a credential works when it does not. `meta: reset_connection` helps but is +# not reliable when connection arguments differ between the reset and the next task. That false +# PASS is unacceptable on a check whose entire purpose is to prove the operator is not locked out, +# and it has been observed on these hosts: a rotation re-run "succeeded" against an old key that +# had already been deleted, and only failed once multiplexing was disabled. +# +# ControlMaster=no plus ControlPath=none makes socket reuse impossible, so the exit code means +# exactly what it appears to mean. IdentitiesOnly=yes stops ssh quietly falling back to an agent +# key and passing with a different credential than the one under test. BatchMode=yes stops it +# hanging on a passphrase or password prompt. +# +# Callers pass: +# ssh_key_rotation_probe_key - path to the private key to test, on the control node +# ssh_key_rotation_probe_label - short description used in messages, e.g. "old key" +# Callers may pass: +# ssh_key_rotation_probe_required - when true (default) a failed probe fails the task; when false +# the caller inspects ssh_key_rotation_probe_result itself +# +# Sets ssh_key_rotation_probe_result, whose .rc is 0 only if that key genuinely authenticated. + +- name: "Probe | Unmultiplexed authentication test for the {{ ssh_key_rotation_probe_label }}" + ansible.builtin.command: + argv: + - ssh + - -i + - "{{ ssh_key_rotation_probe_key }}" + - -o + - IdentitiesOnly=yes + # This probe exists to answer one question: does THIS KEY authenticate. The operator's + # ~/.ssh/config is still honoured (ProxyJump, per-host User, and so on must keep working), + # but a config that turns public key authentication off - or prefers passwords - would make + # the probe fail regardless of whether the key is good, reporting a lockout that has not + # happened. Observed exactly that against a workstation whose ssh_config carried + # `PubkeyAuthentication no` for the target subnet. Forcing these two makes the result depend + # on the key and the server, not on the client's preferences. + - -o + - PubkeyAuthentication=yes + - -o + - PreferredAuthentications=publickey + - -o + - BatchMode=yes + - -o + - ControlMaster=no + - -o + - ControlPath=none + - -o + - StrictHostKeyChecking=accept-new + - -o + - ConnectTimeout=15 + - -p + - "{{ ansible_port | default(22) }}" + - "{{ ssh_key_rotation_target_user }}@{{ ansible_host | default(inventory_hostname) }}" + - "true" + delegate_to: localhost + become: false + changed_when: false + failed_when: false + register: ssh_key_rotation_probe_result + +- name: "Probe | Assert successful authentication by the {{ ssh_key_rotation_probe_label }}" + when: ssh_key_rotation_probe_required | default(true) | bool + ansible.builtin.assert: + that: ssh_key_rotation_probe_result.rc == 0 + fail_msg: >- + The {{ ssh_key_rotation_probe_label }} ({{ ssh_key_rotation_probe_key }}) could NOT + authenticate as {{ ssh_key_rotation_target_user }} on this host. ssh exited + {{ ssh_key_rotation_probe_result.rc }}: + {{ ssh_key_rotation_probe_result.stderr | default('(no stderr)') | trim }} + success_msg: >- + The {{ ssh_key_rotation_probe_label }} authenticated on a fresh, unmultiplexed connection. diff --git a/roles/ssh_key_rotation/tasks/resolve_sshd_strategy.yml b/roles/ssh_key_rotation/tasks/resolve_sshd_strategy.yml new file mode 100644 index 0000000..49b5eb5 --- /dev/null +++ b/roles/ssh_key_rotation/tasks/resolve_sshd_strategy.yml @@ -0,0 +1,34 @@ +--- + +# Decides HOW this host's sshd will be configured, and is included by both install.yml and +# verify.yml so the two stages can never disagree. +# +# Included by verify as well as install rather than relying on install's facts carrying over: they +# do persist between plays of one run, but the verify stage can be run standalone via tasks_from, +# and --limit can exclude a host from the install play. Recomputing costs one grep and removes a +# whole class of "undefined variable" failure - which, inside a rescue:, aborts the rescue and +# leaves a half-changed host with no rollback at all. +# +# Sets: +# ssh_key_rotation_use_dropin - true when sshd reads /etc/ssh/sshd_config.d/ +# ssh_key_rotation_sshd_dropin_effective_prefix - numeric prefix the drop-ins are written with + +- name: Resolve sshd strategy | Find sshd_config's Include and Match directives + ansible.builtin.include_tasks: resolve_sshd_anchor.yml + +# An `Include` line means OpenSSH 8.2 or newer, which is every currently supported distribution. +# Where it exists the role configures itself entirely through drop-ins and never edits the +# administrator's sshd_config, so undoing a run is deleting a file rather than restoring theirs. +- name: Resolve sshd strategy | Decide between drop-ins and editing sshd_config + ansible.builtin.set_fact: + ssh_key_rotation_use_dropin: "{{ ssh_key_rotation_sshd_anchors.stdout is search('(?im)^\\d+:\\s*Include\\s') }}" + ssh_key_rotation_sshd_dropin_effective_prefix: >- + {{ '01' if (ssh_key_rotation_manage_sshd_dropin | bool) else ssh_key_rotation_sshd_dropin_prefix }} + +- name: Resolve sshd strategy | Warn when this host is too old for drop-ins + when: not (ssh_key_rotation_use_dropin | bool) + ansible.builtin.debug: + msg: >- + /etc/ssh/sshd_config on this host has no `Include` line, so this OpenSSH predates 8.2 and + cannot read /etc/ssh/sshd_config.d/. Falling back to a marked block inside sshd_config, which + is backed up first. Rolling that back restores a file rather than deleting one. diff --git a/roles/ssh_key_rotation/tasks/sshd_dropin_write.yml b/roles/ssh_key_rotation/tasks/sshd_dropin_write.yml new file mode 100644 index 0000000..c4aa16b --- /dev/null +++ b/roles/ssh_key_rotation/tasks/sshd_dropin_write.yml @@ -0,0 +1,92 @@ +--- + +# Writes one of this role's sshd drop-ins, records in the host's state fact whether this run +# CREATED it or found one already there, and validates the merged configuration before anything +# reloads. Shared by install.yml and verify.yml so both stages behave identically. +# +# Callers pass: +# ssh_key_rotation_dropin_name - bare filename, e.g. "ssh-key-rotation-lockdown.conf" +# ssh_key_rotation_dropin_content - the file body (already rendered) +# ssh_key_rotation_dropin_key - key under `dropins` in the state fact, e.g. "lockdown" +# +# The created-vs-pre-existing distinction is what lets a rollback undo the right thing: a file we +# created must be deleted, while one that already existed must be restored as it was. Deleting a +# pre-existing drop-in would destroy configuration this role never owned. + +- name: Dropin | Resolve this drop-in's full path + ansible.builtin.set_fact: + ssh_key_rotation_dropin_path: >- + {{ ssh_key_rotation_sshd_dropin_dir }}/{{ ssh_key_rotation_sshd_dropin_effective_prefix }}-{{ ssh_key_rotation_dropin_name }} + +- name: Dropin | Check whether this drop-in already exists + ansible.builtin.stat: + path: "{{ ssh_key_rotation_dropin_path }}" + become: true + register: ssh_key_rotation_dropin_stat + +- name: Dropin | Back up a pre-existing drop-in before overwriting it + when: ssh_key_rotation_dropin_stat.stat.exists + ansible.builtin.copy: + src: "{{ ssh_key_rotation_dropin_path }}" + dest: "{{ ssh_key_rotation_dropin_path }}.bak-{{ ansible_facts.date_time.iso8601_basic_short }}" + remote_src: true + mode: preserve + become: true + register: ssh_key_rotation_dropin_backup + +# Recorded BEFORE the write, so that if the write or the validation below fails, the rescue still +# knows whether this file was ours to delete. +- name: Dropin | Record this drop-in in the host's rotation state + ansible.builtin.include_tasks: state_write.yml + # Built as one Jinja expression, not as YAML keys. A "{{ }}" used as a mapping KEY is not + # templated - it is stored literally - so the nested form recorded a drop-in called + # "{{ ssh_key_rotation_dropin_key }}" and the rollback would never find it. + vars: + ssh_key_rotation_state: >- + {{ {'dropins': {ssh_key_rotation_dropin_key: { + 'path': ssh_key_rotation_dropin_path, + 'created_by_role': (not ssh_key_rotation_dropin_stat.stat.exists), + 'backup': (ssh_key_rotation_dropin_backup.dest | default(none)), + }}} }} + +# No `validate:` here on purpose. `sshd -t -f ` checks the fragment as if it were a whole +# config - it fills in defaults and proves nothing about the merged result - and copy's validate +# runs against a controller-side temp path that SELinux or permissions can reject outright. The +# authoritative check is `sshd -t` on the real merged configuration, immediately below, before any +# handler is allowed to fire. +- name: Dropin | Write the drop-in + ansible.builtin.copy: + dest: "{{ ssh_key_rotation_dropin_path }}" + content: "{{ ssh_key_rotation_dropin_content }}" + mode: "0644" + owner: root + group: root + become: true + register: ssh_key_rotation_dropin_written + notify: Reload sshd + +- name: Dropin | Validate the MERGED sshd configuration with this drop-in in place + ansible.builtin.command: sshd -t + become: true + changed_when: false + failed_when: false + register: ssh_key_rotation_dropin_validate + +# Self-healing: an invalid merged config is removed here rather than left for the rescue, because +# it must never survive long enough for a reload to pick it up. A reload with a broken config fails +# without dropping the listener, but leaving the file in place would break the NEXT legitimate +# reload by anything else on the system. +- name: Dropin | Remove the drop-in again if it broke the merged configuration + when: ssh_key_rotation_dropin_validate.rc != 0 + ansible.builtin.file: + path: "{{ ssh_key_rotation_dropin_path }}" + state: absent + become: true + +- name: Dropin | Fail if this drop-in produced an invalid sshd configuration + when: ssh_key_rotation_dropin_validate.rc != 0 + ansible.builtin.fail: + msg: >- + Writing {{ ssh_key_rotation_dropin_path }} made the merged sshd configuration invalid + ({{ ssh_key_rotation_dropin_validate.stderr | default('no stderr') | trim }}). The drop-in has + been removed again and sshd was never reloaded, so the running configuration is untouched. diff --git a/roles/ssh_key_rotation/tasks/state_read.yml b/roles/ssh_key_rotation/tasks/state_read.yml new file mode 100644 index 0000000..ef7a863 --- /dev/null +++ b/roles/ssh_key_rotation/tasks/state_read.yml @@ -0,0 +1,29 @@ +--- + +# Loads this host's recorded rotation state into ssh_key_rotation_recorded. +# +# Deliberately re-runnable, and re-run at the START OF THE RESCUE as well as at the start of the +# verify stage. Reading it once up front is not enough: the lock-down drop-in is recorded partway +# through the stage, so a snapshot taken beforehand knows nothing about it and the rollback would +# leave that file behind - undoing the run everywhere except the one setting the stage exists to +# apply. Observed exactly that before this was split out. +# +# The file is untrusted input: it lives on the target and an aborted run, or a person, could have +# left it empty, truncated or hand-edited. Anything that is not a JSON object is read as {}, which +# degrades to "no recorded state" rather than raising - and raising inside a rescue: aborts the +# rescue, cancelling the rollback entirely. + +- name: State | Read this host's recorded rotation state + ansible.builtin.slurp: + src: /etc/ansible/facts.d/ssh_key_rotation.fact + become: true + failed_when: false + register: ssh_key_rotation_state_raw + +- name: State | Parse the recorded rotation state + ansible.builtin.set_fact: + ssh_key_rotation_recorded: >- + {{ (ssh_key_rotation_state_raw.content | b64decode | from_json) + if (ssh_key_rotation_state_raw.content is defined + and (ssh_key_rotation_state_raw.content | b64decode | trim).startswith('{')) + else {} }} diff --git a/roles/ssh_key_rotation/tasks/state_write.yml b/roles/ssh_key_rotation/tasks/state_write.yml new file mode 100644 index 0000000..cd482e3 --- /dev/null +++ b/roles/ssh_key_rotation/tasks/state_write.yml @@ -0,0 +1,66 @@ +--- + +# Records what this run actually changed on THIS host, as a local fact under +# /etc/ansible/facts.d, so later stages can undo exactly that and nothing else. +# +# Why a file on the target rather than registered variables. The verify stage's rollback needs to +# know what the install stage did - which drop-ins it created, where it backed authorized_keys up +# to, whether it changed the crypto policy. Registered variables do survive between plays for the +# same host, but relying on them makes the rollback fail in ways that are worse than not having +# it: verify can be run standalone via tasks_from, --limit can exclude a host from the install +# play, and install's backup task is conditional, so the variable can be defined but empty. An +# undefined variable inside a rescue: aborts the rescue, which would leave a half-changed host +# with no rollback at all. +# +# Hosts genuinely differ - one has a cloud-init drop-in, the next runs FIPS, a third already has a +# file at our drop-in path - so this state has to be per host, and the host is the only place it +# stays true. +# +# It also outlives the connection, which is what makes a separate break-glass recovery run +# possible: reconnect later, read what the last run changed, undo precisely that. +# +# Callers pass ssh_key_rotation_state (a dict). Anything not passed is preserved from the existing +# fact, so install can record its half and verify can add to it without either clobbering the other. + +- name: State | Ensure the local facts directory exists + ansible.builtin.file: + path: /etc/ansible/facts.d + state: directory + mode: "0755" + owner: root + group: root + become: true + register: ssh_key_rotation_factsd_dir + +- name: State | Read any existing rotation state so this write merges rather than replaces + ansible.builtin.slurp: + src: /etc/ansible/facts.d/ssh_key_rotation.fact + become: true + failed_when: false + register: ssh_key_rotation_state_existing + +# Treated as untrusted input: it is a file on the target that a previous aborted run, or a human, +# could have left in any shape, so an empty or non-JSON file is read as {} rather than acted on. +# +# startswith('{') rather than a regex on purpose. This is a YAML folded scalar, which performs NO +# escape processing, so a pattern written as '^\\s*\\{' arrives at Jinja as a literal backslash and +# never matches - which silently made every read look like "no prior state" and turned each write +# into a full overwrite, losing the install stage's record by the time verify needed it. +- name: State | Merge the new state onto whatever was already recorded + vars: + ssh_key_rotation_state_prior: >- + {{ (ssh_key_rotation_state_existing.content | b64decode | from_json) + if (ssh_key_rotation_state_existing.content is defined + and (ssh_key_rotation_state_existing.content | b64decode | trim).startswith('{')) + else {} }} + ansible.builtin.set_fact: + ssh_key_rotation_state_merged: "{{ ssh_key_rotation_state_prior | combine(ssh_key_rotation_state, recursive=True) }}" + +- name: State | Write the rotation state fact + ansible.builtin.copy: + dest: /etc/ansible/facts.d/ssh_key_rotation.fact + content: "{{ ssh_key_rotation_state_merged | to_nice_json }}\n" + mode: "0600" + owner: root + group: root + become: true From 1e0c895f8c3f75ba9c3e16d938f562e744a32a71 Mon Sep 17 00:00:00 2001 From: Mark Bolwell Date: Thu, 6 Aug 2026 19:52:27 +0100 Subject: [PATCH 14/18] Test coverage for non-root rotation Signed-off-by: Mark Bolwell --- .github/workflows/molecule.yml | 6 +- extensions/molecule/nonroot/converge.yml | 103 +++++++++++++++ extensions/molecule/nonroot/molecule.yml | 79 ++++++++++++ extensions/molecule/nonroot/prepare.yml | 26 ++++ extensions/molecule/nonroot/verify.yml | 145 ++++++++++++++++++++++ extensions/molecule/resources/prepare.yml | 106 +++++++++++++--- extensions/molecule/rollback/converge.yml | 10 +- extensions/molecule/rollback/molecule.yml | 10 ++ extensions/molecule/rollback/prepare.yml | 3 + extensions/molecule/rollback/verify.yml | 66 +++++++++- 10 files changed, 527 insertions(+), 27 deletions(-) create mode 100644 extensions/molecule/nonroot/converge.yml create mode 100644 extensions/molecule/nonroot/molecule.yml create mode 100644 extensions/molecule/nonroot/prepare.yml create mode 100644 extensions/molecule/nonroot/verify.yml diff --git a/.github/workflows/molecule.yml b/.github/workflows/molecule.yml index 8eb6e93..a267be3 100644 --- a/.github/workflows/molecule.yml +++ b/.github/workflows/molecule.yml @@ -20,7 +20,11 @@ jobs: strategy: fail-fast: false matrix: - scenario: [default, rollback] + # nonroot is not a variant of default for its own sake: default and rollback both enter + # the containers as root and rotate root, so a file left owned by root is the correct + # outcome there and an ownership regression cannot fail them. nonroot is the only + # scenario that can catch it. + scenario: [default, rollback, nonroot] steps: - name: Check out repository uses: actions/checkout@v7 diff --git a/extensions/molecule/nonroot/converge.yml b/extensions/molecule/nonroot/converge.yml new file mode 100644 index 0000000..b5a1b78 --- /dev/null +++ b/extensions/molecule/nonroot/converge.yml @@ -0,0 +1,103 @@ +--- + +# Runs the collection's real entry point (playbooks/rotate.yml, all 3 stages) against the +# generated inventory/vars from prepare.yml -- this is what proves the shipped playbook works +# end to end, not just the role in isolation. Then runs it again, unchanged, to prove +# idempotency: a second identical rotation should report changed=0 across the board. + +- name: Converge | Run the real rotation playbook end to end (first pass) + hosts: localhost + connection: local + gather_facts: false + vars: + ssh_key_rotation_test_dir: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}" + ssh_key_rotation_repo_root: "{{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') }}" + tasks: + - name: Converge | First rotation run + ansible.builtin.command: >- + ansible-playbook -i {{ ssh_key_rotation_test_dir }}/rotation_inventory.ini + {{ ssh_key_rotation_repo_root }}/playbooks/rotate.yml + -e @{{ ssh_key_rotation_test_dir }}/rotation_vars.yml + environment: + ANSIBLE_ROLES_PATH: "{{ ssh_key_rotation_repo_root }}/roles" + ANSIBLE_HOST_KEY_CHECKING: "false" + changed_when: true + register: ssh_key_rotation_first_run + + - name: Converge | Show first run output + ansible.builtin.debug: + var: ssh_key_rotation_first_run.stdout_lines + + - name: Converge | Second (idempotency) rotation run with identical vars + ansible.builtin.command: >- + ansible-playbook -i {{ ssh_key_rotation_test_dir }}/rotation_inventory.ini + {{ ssh_key_rotation_repo_root }}/playbooks/rotate.yml + -e @{{ ssh_key_rotation_test_dir }}/rotation_vars.yml + environment: + ANSIBLE_ROLES_PATH: "{{ ssh_key_rotation_repo_root }}/roles" + ANSIBLE_HOST_KEY_CHECKING: "false" + # JSON callback so idempotency can be judged per task below, rather than from the PLAY + # RECAP's per-host counts (which can't say WHICH task changed). Warnings still go to + # stderr, so stdout stays valid JSON. + ANSIBLE_STDOUT_CALLBACK: json + changed_when: true + register: ssh_key_rotation_second_run + + # Two kinds of task are expected to change on every run and are excluded here; anything else + # changing is a real idempotency regression, which is what this splits apart. + # + # "Back up ..." writes to a dest containing a timestamp, so it produces a new file each run. + # That is deliberate: each run must snapshot the state before that run, and a fixed filename + # would overwrite the original pre-rotation backup. + # + # "Write the rotation state fact" records run_id, a fresh timestamp per run, so its content + # differs every time by design. That file is what lets a rollback undo exactly what this run + # did, and what makes recovery possible after the connection is gone, so it cannot be made + # content-stable without losing the thing it is for. + - name: Converge | Split the second run's changed tasks into expected backups vs unexpected + vars: + ssh_key_rotation_second_run_json: "{{ ssh_key_rotation_second_run.stdout | from_json }}" + ansible.builtin.set_fact: + ssh_key_rotation_unexpected_changed: >- + {%- set found = [] -%} + {%- for play in ssh_key_rotation_second_run_json.plays -%} + {%- for task in play.tasks -%} + {%- for host, result in task.hosts.items() -%} + {%- if (result.changed | default(false)) and 'Back up' not in task.task.name + and 'rotation state fact' not in task.task.name -%} + {%- set _ = found.append(task.task.name ~ ' on ' ~ host) -%} + {%- endif -%} + {%- endfor -%} + {%- endfor -%} + {%- endfor -%} + {{ found | unique | list }} + ssh_key_rotation_expected_backup_changed: >- + {%- set found = [] -%} + {%- for play in ssh_key_rotation_second_run_json.plays -%} + {%- for task in play.tasks -%} + {%- for host, result in task.hosts.items() -%} + {%- if (result.changed | default(false)) and ('Back up' in task.task.name + or 'rotation state fact' in task.task.name) -%} + {%- set _ = found.append(task.task.name) -%} + {%- endif -%} + {%- endfor -%} + {%- endfor -%} + {%- endfor -%} + {{ found | unique | list }} + + - name: Converge | Show what the second (idempotency) run changed + ansible.builtin.debug: + msg: + expected_timestamped_backups: "{{ ssh_key_rotation_expected_backup_changed }}" + unexpected_changes: "{{ ssh_key_rotation_unexpected_changed }}" + + - name: Converge | Assert the second run changed nothing beyond the timestamped backups + ansible.builtin.assert: + that: ssh_key_rotation_unexpected_changed | length == 0 + fail_msg: >- + Second identical rotation run changed task(s) other than the timestamped backups, so + rotation is not idempotent: {{ ssh_key_rotation_unexpected_changed }} + success_msg: >- + Second identical rotation run was idempotent: the only changes were the + {{ ssh_key_rotation_expected_backup_changed | length }} timestamped backup file(s), + which are re-created on every run by design. diff --git a/extensions/molecule/nonroot/molecule.yml b/extensions/molecule/nonroot/molecule.yml new file mode 100644 index 0000000..0395f97 --- /dev/null +++ b/extensions/molecule/nonroot/molecule.yml @@ -0,0 +1,79 @@ +--- + +driver: + name: docker +platforms: + # The first platform carries the settings every platform shares; the rest merge it and + # override only name/image. The Dockerfile itself is shared across scenarios too, hence the + # relative path (the docker driver resolves `dockerfile` against this scenario directory). + - &platform_defaults + name: ssh-key-rotation-nonroot-ubuntu2204 + image: geerlingguy/docker-ubuntu2204-ansible:latest + dockerfile: ../resources/Dockerfile.j2 + command: /usr/sbin/init + privileged: true + cgroupns_mode: host + pre_build_image: false + tmpfs: + - /run + - /run/lock + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:rw + groups: + - rotate + - <<: *platform_defaults + name: ssh-key-rotation-nonroot-rockylinux9 + image: geerlingguy/docker-rockylinux9-ansible:latest + - <<: *platform_defaults + name: ssh-key-rotation-nonroot-rockylinux10 + image: geerlingguy/docker-rockylinux10-ansible:latest +dependency: + # Mirrors exactly what CI used to run as a separate step, so a local `molecule test` and a + # CI run install the same collection dependencies. Molecule chdirs to the project directory + # (the collection root) before running a sequence, so this relative path is correct. + name: shell + command: ansible-galaxy install -r requirements.yml +provisioner: + name: ansible + env: + # This scenario's converge shells out to playbooks/rotate.yml with its own absolute roles + # path, so nothing here strictly needs this today. It is set anyway to keep both scenarios + # identical and to stay correct if a play here ever includes the role directly. Two traps: + # the collection's own ansible.cfg (roles_path = ./roles) is NOT consulted, because + # Molecule generates its own ansible.cfg and points ANSIBLE_CONFIG at it; and Molecule + # runs with the project directory as cwd, not the scenario directory, so a + # scenario-relative path like ../../../roles resolves outside the repo entirely. + # MOLECULE_PROJECT_DIRECTORY is the collection root, and the `.` fallback is the same + # thing whenever it isn't exported, since Molecule chdirs there before running a sequence. + ANSIBLE_ROLES_PATH: "${MOLECULE_PROJECT_DIRECTORY:-.}/roles" + inventory: + group_vars: + # The docker connection plugin doesn't invoke a login shell, so `~` in the default + # remote_tmp path doesn't reliably expand for the "rotate" containers. Pin it to the + # absolute path resources/Dockerfile.j2 already creates for root in the image. Scoped to + # this group only - localhost (used by molecule's own internal playbooks and the + # key-generation play) must keep its normal, non-root remote_tmp. + rotate: + ansible_remote_tmp: /root/.ansible/tmp + # These containers are entered as root. Saying so explicitly means Ansible skips + # become for the role's `become: true` tasks (become_user root == remote_user root) + # instead of shelling out to sudo. Without this, sudo runs and fails on images whose + # pam/shadow setup can't serve an account lookup inside a container (seen on + # rockylinux9: "PAM account management error: Authentication service cannot retrieve + # authentication info"), which has nothing to do with what these tests are proving. + ansible_user: root +scenario: + test_sequence: + - dependency + # Leading destroy: without it a container left behind by an aborted run is reused, and its + # authorized_keys already holds the rotated-in key. That silently breaks this suite's core + # premise (only the seeded old key is authorized before converge runs). + - destroy + - syntax + - create + - prepare + - converge + - verify + - destroy +verifier: + name: ansible diff --git a/extensions/molecule/nonroot/prepare.yml b/extensions/molecule/nonroot/prepare.yml new file mode 100644 index 0000000..40df3b6 --- /dev/null +++ b/extensions/molecule/nonroot/prepare.yml @@ -0,0 +1,26 @@ +--- + +# Same shape as the default scenario, but rotates a NON-root user. That difference is the whole +# point of this scenario: the other scenarios enter the containers as root and rotate root, so +# "the file ended up owned by root" is the correct result there and any ownership bug is +# invisible. With a non-root target the role still does its file work under become (as root), +# so every authorized_keys it writes or restores must be explicitly chowned back to the target +# user - otherwise sshd's StrictModes ignores the file and refuses every key for that user. +# +# The user's primary group is deliberately NOT the same name as the user (see resources/prepare.yml), +# so the role cannot pass by assuming group == username. + +- name: Prepare | Set up the containers as non-root rotation targets + ansible.builtin.import_playbook: ../resources/prepare.yml + vars: + ssh_key_rotation_test_user: svcrotate + ssh_key_rotation_test_keys: + - name: old_key + type: rsa + bits: 3072 + - name: new_key + type: ed25519 + ssh_key_rotation_test_seed_key: old_key + ssh_key_rotation_test_rotation_vars: + old: old_key + new: new_key diff --git a/extensions/molecule/nonroot/verify.yml b/extensions/molecule/nonroot/verify.yml new file mode 100644 index 0000000..bcb9967 --- /dev/null +++ b/extensions/molecule/nonroot/verify.yml @@ -0,0 +1,145 @@ +--- + +# Same guarantees as the default scenario's verify, re-stated for a NON-root target user, plus +# the ownership assertions that only mean anything when the target isn't root. + +- name: Verify | Inspect the post-rotation state on each container + hosts: rotate + gather_facts: false + vars: + ssh_key_rotation_test_dir: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}" + ssh_key_rotation_test_user: svcrotate + tasks: + # As in the default scenario: this play reaches the containers over the Molecule docker + # connection, which ignores ansible_ssh_private_key_file, so this is a liveness check only. + # The real key-authentication proof is the second play, which drives an actual ssh client. + - name: Verify | Container is reachable for inspection + ansible.builtin.ping: + + - name: Verify | Stat the rotated user's authorized_keys + ansible.builtin.stat: + path: "/home/{{ ssh_key_rotation_test_user }}/.ssh/authorized_keys" + register: ssh_key_rotation_ak_stat + + # The regression guard this scenario exists for. The role does this work under become, so a + # copy that preserves only the MODE leaves the file owned by root. sshd's StrictModes then + # ignores it outright and refuses every key for this user - a full lockout that still looks + # like a successful run in the play recap. + - name: Verify | authorized_keys is owned by the rotated user, not root + ansible.builtin.assert: + that: + - ssh_key_rotation_ak_stat.stat.exists + - ssh_key_rotation_ak_stat.stat.pw_name == ssh_key_rotation_test_user + - not ssh_key_rotation_ak_stat.stat.wgrp + - not ssh_key_rotation_ak_stat.stat.woth + fail_msg: >- + authorized_keys is owned by {{ ssh_key_rotation_ak_stat.stat.pw_name | default('(missing)') }} + (mode {{ ssh_key_rotation_ak_stat.stat.mode | default('n/a') }}), not + {{ ssh_key_rotation_test_user }}. sshd StrictModes will ignore it and refuse every key. + success_msg: "authorized_keys is owned by {{ ssh_key_rotation_test_user }} and satisfies StrictModes." + + # Backups land beside authorized_keys in the same .ssh directory and are written by the same + # become-elevated tasks, so they are checked too - a root-owned backup is what a later + # rollback would restore FROM, reintroducing the lockout at the worst possible moment. + - name: Verify | Find the backups the role left behind + ansible.builtin.find: + paths: "/home/{{ ssh_key_rotation_test_user }}/.ssh" + patterns: "authorized_keys.bak-*" + register: ssh_key_rotation_ak_backups + + - name: Verify | Every authorized_keys backup is owned by the rotated user + ansible.builtin.assert: + that: ssh_key_rotation_ak_backups.files | rejectattr('pw_name', 'equalto', ssh_key_rotation_test_user) | list | length == 0 + fail_msg: >- + These authorized_keys backups are not owned by {{ ssh_key_rotation_test_user }}: + {{ ssh_key_rotation_ak_backups.files | rejectattr('pw_name', 'equalto', ssh_key_rotation_test_user) + | map(attribute='path') | list }}. Restoring one of these during a rollback would + leave the user locked out. + success_msg: "All {{ ssh_key_rotation_ak_backups.files | length }} backup(s) are correctly owned." + + - name: Verify | Read authorized_keys + ansible.builtin.slurp: + src: "/home/{{ ssh_key_rotation_test_user }}/.ssh/authorized_keys" + register: ssh_key_rotation_authorized_keys_content + + # Literal `in`, never `is search`: a base64 key blob routinely contains `+`, which `search` + # would treat as a regex quantifier. On the negative assertion that is actively dangerous, + # since `not (no match)` is True and would report the old key as removed while it is present. + - name: Verify | New key is present in authorized_keys + ansible.builtin.assert: + that: >- + lookup('file', ssh_key_rotation_test_dir + '/new_key.pub').split()[1] + in (ssh_key_rotation_authorized_keys_content.content | b64decode) + fail_msg: "The new key is not in authorized_keys after rotation." + success_msg: "New key is present in authorized_keys." + + - name: Verify | Old key has been removed from authorized_keys + ansible.builtin.assert: + that: >- + lookup('file', ssh_key_rotation_test_dir + '/old_key.pub').split()[1] + not in (ssh_key_rotation_authorized_keys_content.content | b64decode) + fail_msg: "The old key is STILL in authorized_keys after rotation - it should have been removed." + success_msg: "Old key has been removed from authorized_keys." + + - name: Verify | Check effective sshd configuration + ansible.builtin.command: sshd -T + become: true + changed_when: false + register: ssh_key_rotation_sshd_effective + + - name: Verify | sshd_config reflects the role's default safety settings + ansible.builtin.assert: + that: + - "'pubkeyauthentication yes' in ssh_key_rotation_sshd_effective.stdout | lower" + - "'passwordauthentication no' in ssh_key_rotation_sshd_effective.stdout | lower" + - "'kbdinteractiveauthentication no' in ssh_key_rotation_sshd_effective.stdout | lower" + +- name: Verify | Confirm real SSH auth for the non-root user after rotation + hosts: rotate + gather_facts: false + vars: + ssh_key_rotation_test_dir: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}" + ssh_key_rotation_test_user: svcrotate + ssh_key_rotation_verify_ssh_opts: >- + -o BatchMode=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null + -o IdentitiesOnly=yes -o ConnectTimeout=15 + tasks: + - name: Verify | Discover this container's IP address for the real SSH checks + ansible.builtin.command: hostname -I + changed_when: false + register: ssh_key_rotation_verify_container_ip + + # Positive control, first on purpose: a host where sshd rejects EVERY key (the exact + # StrictModes lockout this scenario guards against) would otherwise satisfy the old-key + # assertion below and pass while all access had been lost. + - name: Verify | NEW key authenticates over real SSH as the non-root user (positive control) + ansible.builtin.command: >- + ssh -i {{ ssh_key_rotation_test_dir }}/new_key {{ ssh_key_rotation_verify_ssh_opts }} + {{ ssh_key_rotation_test_user }}@{{ ssh_key_rotation_verify_container_ip.stdout.split() | first }} true + delegate_to: localhost + changed_when: false + + - name: Verify | Attempt a real SSH connection with the OLD key (must be refused) + ansible.builtin.command: >- + ssh -i {{ ssh_key_rotation_test_dir }}/old_key {{ ssh_key_rotation_verify_ssh_opts }} + {{ ssh_key_rotation_test_user }}@{{ ssh_key_rotation_verify_container_ip.stdout.split() | first }} true + delegate_to: localhost + changed_when: false + failed_when: false + register: ssh_key_rotation_old_key_ssh + + # Both conditions matter: a non-zero rc alone is also satisfied by a timeout or a refused + # connection, which would hide a broken environment behind a passing assertion. Requiring + # sshd's own "Permission denied" proves the daemon answered and rejected the key. + - name: Verify | Assert sshd refused the OLD key + ansible.builtin.assert: + that: + - ssh_key_rotation_old_key_ssh.rc != 0 + - "'permission denied' in (ssh_key_rotation_old_key_ssh.stderr | default('') | lower)" + fail_msg: >- + Old key still authenticates after rotation - it should have been removed. + ssh exited {{ ssh_key_rotation_old_key_ssh.rc }}; + stderr: {{ ssh_key_rotation_old_key_ssh.stderr | default('(none)') }} + success_msg: >- + sshd refused the old key while the new key still authenticates as + {{ ssh_key_rotation_test_user }} - rotation removed old access as intended. diff --git a/extensions/molecule/resources/prepare.yml b/extensions/molecule/resources/prepare.yml index f39b46f..8e50cf2 100644 --- a/extensions/molecule/resources/prepare.yml +++ b/extensions/molecule/resources/prepare.yml @@ -19,6 +19,15 @@ # rotation_vars.yml, the single extra-vars file used by a # converge that shells out to playbooks/rotate.yml. Scenarios # that drive the role's stages directly don't need it. +# ssh_key_rotation_test_user Unix user to rotate. Defaults to "root", which is what the +# containers are entered as. Set it to a non-root name to +# exercise the case the root-only scenarios structurally +# cannot: with become in play, the role's file operations run +# as root, so any authorized_keys it writes for a NON-root +# user must be explicitly chowned back. If it isn't, sshd's +# StrictModes ignores the file and every key is refused. That +# whole class of bug is invisible when target_user is root, +# because root-owned is then the correct outcome. - name: Prepare | Generate ephemeral keys and the test inventory on the control node hosts: localhost @@ -52,7 +61,7 @@ old_public_key_file: "{{ ssh_key_rotation_test_dir }}/{{ ssh_key_rotation_test_old }}.pub" new_private_key: "{{ ssh_key_rotation_test_dir }}/{{ ssh_key_rotation_test_new }}" new_public_key_file: "{{ ssh_key_rotation_test_dir }}/{{ ssh_key_rotation_test_new }}.pub" - ssh_key_rotation_target_user: root + ssh_key_rotation_target_user: {{ ssh_key_rotation_test_user | default('root') }} - name: Prepare | Seed the pre-existing key into each container's authorized_keys hosts: rotate @@ -76,9 +85,36 @@ line: 'PermitRootLogin prohibit-password' notify: Restart sshd for prepare + # Only for non-root scenarios. Given its own primary group deliberately named differently + # from the username, so the role cannot get ownership right by assuming group == user. + - name: Prepare | Create the non-root rotation target group + when: (ssh_key_rotation_test_user | default('root')) != 'root' + ansible.builtin.group: + name: "{{ ssh_key_rotation_test_user }}grp" + state: present + + - name: Prepare | Create the non-root rotation target user + when: (ssh_key_rotation_test_user | default('root')) != 'root' + ansible.builtin.user: + name: "{{ ssh_key_rotation_test_user }}" + group: "{{ ssh_key_rotation_test_user }}grp" + shell: /bin/bash + create_home: true + state: present + + # The role needs become to edit sshd_config; the rotation user must therefore be able to + # sudo without a password, exactly as a real managed account would be. + - name: Prepare | Grant the non-root rotation target passwordless sudo + when: (ssh_key_rotation_test_user | default('root')) != 'root' + ansible.builtin.copy: + dest: "/etc/sudoers.d/90-{{ ssh_key_rotation_test_user }}-molecule" + content: "{{ ssh_key_rotation_test_user }} ALL=(ALL) NOPASSWD:ALL\n" # pragma: allowlist secret + mode: "0440" + validate: "visudo -cf %s" + - name: Prepare | Install the ephemeral seed public key as pre-existing authorized access ansible.posix.authorized_key: - user: root + user: "{{ ssh_key_rotation_test_user | default('root') }}" key: "{{ lookup('file', ssh_key_rotation_test_dir + '/' + ssh_key_rotation_test_seed_key + '.pub') }}" state: present @@ -91,26 +127,62 @@ ansible.builtin.set_fact: ssh_key_rotation_container_ip: "{{ ssh_key_rotation_container_ip.stdout.split() | first }}" - - name: Prepare | Ensure the [rotate] group header is present in the generated inventory - ansible.builtin.lineinfile: - path: "{{ ssh_key_rotation_test_dir }}/rotation_inventory.ini" - line: "[rotate]" - insertbefore: BOF - create: true + # Written once, from hostvars, instead of each host appending its own line with lineinfile. + # That earlier approach had every host in the group read-modify-write the SAME file on the + # control node concurrently (delegate_to: localhost, default forks), so a host's line could + # be lost to a classic lost-update race. The symptom was silent and dangerous: the dropped + # container was still created and seeded, but never rotated or verified, and the suite still + # reported success - a regression affecting only that platform would pass CI unnoticed. + - name: Prepare | Write the generated rotation inventory in a single pass + vars: + # Split out purely to keep the generated line below within the linter's length limit; + # each entry still has to render as ONE line of INI. + ssh_key_rotation_inv_user: "{{ ssh_key_rotation_test_user | default('root') }}" + ssh_key_rotation_inv_ssh_args: >- + -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null + ansible.builtin.copy: + dest: "{{ ssh_key_rotation_test_dir }}/rotation_inventory.ini" mode: "0600" + content: | + [rotate] + {% for host in ansible_play_hosts | sort %} + {{ host }} ansible_host={{ hostvars[host].ssh_key_rotation_container_ip }} + {{- ' ansible_user=' ~ ssh_key_rotation_inv_user }} + {{- " ansible_ssh_common_args='" ~ ssh_key_rotation_inv_ssh_args ~ "'" }} + {% endfor %} + {% if (ssh_key_rotation_test_user | default('root')) != 'root' %} + + [rotate:vars] + ansible_become=true + ansible_become_method=sudo + {% endif %} delegate_to: localhost run_once: true - - name: Prepare | Append this host to the generated rotation inventory - ansible.builtin.lineinfile: - path: "{{ ssh_key_rotation_test_dir }}/rotation_inventory.ini" - create: true - mode: "0600" - line: >- - {{ inventory_hostname }} ansible_host={{ ssh_key_rotation_container_ip }} ansible_user=root - ansible_ssh_common_args='-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null' - insertafter: EOF + # The two host lists differ on purpose, which is what gives this assert something real to + # catch: the file above is written from ansible_play_hosts (hosts STILL ACTIVE), while this + # compares against ansible_play_hosts_all (every host Molecule created). A container that + # failed an earlier prepare task - the user or sudoers tasks above, say - drops out of the + # former but not the latter, so it would be silently omitted from the rotation inventory and + # never rotated, while the suite still reported success. Comparing the file against the same + # variable it was generated from would be a tautology that can never fail. + - name: Prepare | Assert every created host made it into the generated inventory + vars: + ssh_key_rotation_inventory_hosts: >- + {{ lookup('file', ssh_key_rotation_test_dir + '/rotation_inventory.ini') + | regex_findall('(?m)^(\S+)\s+ansible_host=') }} + ansible.builtin.assert: + that: ssh_key_rotation_inventory_hosts | sort == ansible_play_hosts_all | sort + fail_msg: >- + Generated rotation inventory lists {{ ssh_key_rotation_inventory_hosts | sort }} + but Molecule created {{ ansible_play_hosts_all | sort }}. The missing host(s) either + failed a prepare task or were lost while writing the file; either way they would be + seeded but never rotated, and the suite would still pass. + success_msg: >- + All {{ ansible_play_hosts_all | length }} created host(s) are present in the + generated rotation inventory. delegate_to: localhost + run_once: true handlers: - name: Restart sshd for prepare diff --git a/extensions/molecule/rollback/converge.yml b/extensions/molecule/rollback/converge.yml index e9d5533..55b3afe 100644 --- a/extensions/molecule/rollback/converge.yml +++ b/extensions/molecule/rollback/converge.yml @@ -42,7 +42,7 @@ new_private_key: "{{ ssh_key_rotation_test_dir }}/key_a" new_public_key_file: "{{ ssh_key_rotation_test_dir }}/key_a.pub" ansible_ssh_private_key_file: "{{ old_private_key }}" - ssh_key_rotation_target_user: root + ssh_key_rotation_target_user: "{{ ssh_key_rotation_rollback_user }}" tasks: - name: Converge | Run role install stage ansible.builtin.include_role: @@ -58,7 +58,7 @@ new_private_key: "{{ ssh_key_rotation_test_dir }}/key_a" new_public_key_file: "{{ ssh_key_rotation_test_dir }}/key_a.pub" ansible_ssh_private_key_file: "{{ new_private_key }}" - ssh_key_rotation_target_user: root + ssh_key_rotation_target_user: "{{ ssh_key_rotation_rollback_user }}" ssh_key_rotation_disable_password_auth: false ssh_key_rotation_disable_kbd_interactive: false tasks: @@ -76,7 +76,7 @@ tasks: - name: Converge | Read authorized_keys after rotation 1 ansible.builtin.slurp: - src: /root/.ssh/authorized_keys + src: "/home/{{ ssh_key_rotation_rollback_user }}/.ssh/authorized_keys" register: ssh_key_rotation_checkpoint_authorized_keys - name: Converge | Save the authorized_keys checkpoint locally @@ -96,7 +96,7 @@ new_private_key: "{{ ssh_key_rotation_test_dir }}/key_b" new_public_key_file: "{{ ssh_key_rotation_test_dir }}/key_b.pub" ansible_ssh_private_key_file: "{{ old_private_key }}" - ssh_key_rotation_target_user: root + ssh_key_rotation_target_user: "{{ ssh_key_rotation_rollback_user }}" tasks: - name: Converge | Run role install stage ansible.builtin.include_role: @@ -112,7 +112,7 @@ new_private_key: "{{ ssh_key_rotation_test_dir }}/key_b" new_public_key_file: "{{ ssh_key_rotation_test_dir }}/key_b.pub" ansible_ssh_private_key_file: "{{ new_private_key }}" - ssh_key_rotation_target_user: root + ssh_key_rotation_target_user: "{{ ssh_key_rotation_rollback_user }}" ssh_key_rotation_sshd_service: "bogus-sshd-molecule-test" tasks: # block/rescue, not `ignore_errors` + `register` on the include_role: ignore_errors does NOT diff --git a/extensions/molecule/rollback/molecule.yml b/extensions/molecule/rollback/molecule.yml index c29efb6..51b4984 100644 --- a/extensions/molecule/rollback/molecule.yml +++ b/extensions/molecule/rollback/molecule.yml @@ -60,6 +60,16 @@ provisioner: # rockylinux9: "PAM account management error: Authentication service cannot retrieve # authentication info"), which has nothing to do with what these tests are proving. ansible_user: root + # The Unix account this scenario rotates. Deliberately NOT root: the rescue block restores + # authorized_keys while running under become (as root), so if that restore does not chown + # the file back, sshd's StrictModes ignores it and every key is refused - the rollback that + # exists to preserve access becomes the thing that destroys it. With a root target that bug + # is invisible, because root-owned is then the correct result. This is the only scenario + # that drives the rescue path, so it is the only place that regression can be caught. + # Note the containers are still ENTERED as root above (that is Molecule's own connection); + # only the rotation target differs. + all: + ssh_key_rotation_rollback_user: svcrotate scenario: test_sequence: - dependency diff --git a/extensions/molecule/rollback/prepare.yml b/extensions/molecule/rollback/prepare.yml index 3220d56..118114b 100644 --- a/extensions/molecule/rollback/prepare.yml +++ b/extensions/molecule/rollback/prepare.yml @@ -10,6 +10,9 @@ - name: Prepare | Set up the containers as rotation targets ansible.builtin.import_playbook: ../resources/prepare.yml vars: + # Must match ssh_key_rotation_rollback_user in this scenario's molecule.yml. Non-root on + # purpose - see the comment there for why the rescue path can only be tested this way. + ssh_key_rotation_test_user: svcrotate ssh_key_rotation_test_keys: - name: seed_key type: rsa diff --git a/extensions/molecule/rollback/verify.yml b/extensions/molecule/rollback/verify.yml index 881d197..ac36e0f 100644 --- a/extensions/molecule/rollback/verify.yml +++ b/extensions/molecule/rollback/verify.yml @@ -30,18 +30,76 @@ - name: Verify | key_a authenticates over real SSH after the rollback ansible.builtin.command: >- ssh -i {{ ssh_key_rotation_test_dir }}/key_a {{ ssh_key_rotation_verify_ssh_opts }} - root@{{ ssh_key_rotation_verify_container_ip.stdout.split() | first }} true + {{ ssh_key_rotation_rollback_user }}@{{ ssh_key_rotation_verify_container_ip.stdout.split() | first }} true delegate_to: localhost changed_when: false - name: Verify | sshd is running - ansible.builtin.command: "systemctl is-active {{ ssh_key_rotation_sshd_service }}" + ansible.builtin.systemd_service: + name: "{{ ssh_key_rotation_sshd_service }}" become: true - changed_when: false + register: ssh_key_rotation_sshd_unit + + - name: Verify | Assert sshd is still active after the rolled-back rotation + ansible.builtin.assert: + that: ssh_key_rotation_sshd_unit.status.ActiveState == 'active' + fail_msg: >- + {{ ssh_key_rotation_sshd_service }} is + {{ ssh_key_rotation_sshd_unit.status.ActiveState }}, not active, after the rollback. + success_msg: "{{ ssh_key_rotation_sshd_service }} is still active after the rollback." + + # THE regression guard for the rescue path. The rollback restores authorized_keys with a copy + # running as root; if that copy does not chown the file back to the rotated user, sshd's + # StrictModes ignores it completely and refuses every key - a full lockout that still looks + # like a clean, successful rollback in the play recap. Checked before the content assertions + # below because restoring the right bytes with the wrong owner is still a locked-out host. + - name: Verify | Stat authorized_keys after the rolled-back rotation + ansible.builtin.stat: + path: "/home/{{ ssh_key_rotation_rollback_user }}/.ssh/authorized_keys" + register: ssh_key_rotation_rollback_ak_stat + + - name: Verify | Rollback restored authorized_keys with the rotated user's ownership intact + ansible.builtin.assert: + that: + - ssh_key_rotation_rollback_ak_stat.stat.exists + - ssh_key_rotation_rollback_ak_stat.stat.pw_name == ssh_key_rotation_rollback_user + - not ssh_key_rotation_rollback_ak_stat.stat.wgrp + - not ssh_key_rotation_rollback_ak_stat.stat.woth + fail_msg: >- + After the rollback, authorized_keys is owned by + {{ ssh_key_rotation_rollback_ak_stat.stat.pw_name | default('(missing file)') }} + (mode {{ ssh_key_rotation_rollback_ak_stat.stat.mode | default('n/a') }}), not + {{ ssh_key_rotation_rollback_user }}. sshd StrictModes will ignore the file and refuse + every key - the rescue block restored the content but locked the user out. + success_msg: >- + Rollback restored authorized_keys still owned by {{ ssh_key_rotation_rollback_user }}; + sshd StrictModes will honour it. + + # Backups matter too: these are what a LATER rollback would restore from, so a root-owned + # backup is a lockout waiting to happen on the next failed rotation. + - name: Verify | Find the backups left behind by the rotations + ansible.builtin.find: + paths: "/home/{{ ssh_key_rotation_rollback_user }}/.ssh" + patterns: "authorized_keys.bak-*" + register: ssh_key_rotation_rollback_ak_backups + + - name: Verify | Every authorized_keys backup is owned by the rotated user + ansible.builtin.assert: + that: >- + ssh_key_rotation_rollback_ak_backups.files + | rejectattr('pw_name', 'equalto', ssh_key_rotation_rollback_user) | list | length == 0 + fail_msg: >- + These authorized_keys backups are not owned by {{ ssh_key_rotation_rollback_user }}: + {{ ssh_key_rotation_rollback_ak_backups.files + | rejectattr('pw_name', 'equalto', ssh_key_rotation_rollback_user) + | map(attribute='path') | list }}. Restoring one of these in a future rollback would + lock the user out. + success_msg: >- + All {{ ssh_key_rotation_rollback_ak_backups.files | length }} backup(s) are correctly owned. - name: Verify | Read authorized_keys after the failed rotation ansible.builtin.slurp: - src: /root/.ssh/authorized_keys + src: "/home/{{ ssh_key_rotation_rollback_user }}/.ssh/authorized_keys" register: ssh_key_rotation_post_failure_authorized_keys - name: Verify | Load the pre-failure checkpoint From 7d035a7ac5d3e737bc589f39db6fcd6a2e6d0844 Mon Sep 17 00:00:00 2001 From: Mark Bolwell Date: Thu, 6 Aug 2026 19:53:10 +0100 Subject: [PATCH 15/18] Docs updates Signed-off-by: Mark Bolwell --- .gitignore | 3 + CONTRIBUTING.md | 17 +- DEVELOPMENT.md | 28 ++- PQC.md | 122 +++++++++++ QUICKSTART.md | 136 ++++++------ README.md | 432 ++++++++++++++++++++++++++------------ rotation_vars.example.yml | 33 +++ 7 files changed, 554 insertions(+), 217 deletions(-) create mode 100644 PQC.md create mode 100644 rotation_vars.example.yml diff --git a/.gitignore b/.gitignore index e48cc38..6d9b161 100644 --- a/.gitignore +++ b/.gitignore @@ -43,3 +43,6 @@ test_* *_ecdsa* *_ed25519* !*.pub.example + +# Personal copy of rotation_vars.example.yml; contains local key paths, not for committing. +rotation_vars.yml diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 31a4cff..de15e91 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -57,10 +57,17 @@ Molecule needs a working Docker or Podman socket. The scenarios pull `geerlinggu | RHEL/Fedora crypto-policy handling | `roles/ssh_key_rotation/tasks/manage_crypto_policy.yml` | | A variable's default value | `roles/ssh_key_rotation/defaults/main.yml` | | How the three stages are wired together | `playbooks/rotate.yml` | -| Functional tests | `extensions/molecule/default/`, `extensions/molecule/rollback/` | +| Functional tests | `extensions/molecule/default/`, `nonroot/`, `rollback/` | +| Shared test setup for all scenarios | `extensions/molecule/resources/prepare.yml` | The role has no working `tasks/main.yml` entry point by design, because each stage authenticates differently. Do not add one. +Everything the role writes to `sshd_config` goes into a marked block placed by +`tasks/resolve_sshd_anchor.yml`, never with a bare `lineinfile`. A `lineinfile` regexp matching a +keyword anywhere in the file replaces the LAST occurrence, and `Match` blocks sit at the end of +`sshd_config`, so it silently edits per-user policy instead of the global section. If you add a +new `sshd_config` setting, put it in the managed block. + ## Running the checks These are the same checks CI runs, so running them locally saves a round trip. @@ -84,10 +91,18 @@ The hooks are the last line of defence against committing key material, which is # Full rotation across three OS images, run twice to prove idempotency molecule test -s default +# The same rotation against a non-root user, to catch file-ownership mistakes +molecule test -s nonroot + # Deliberately breaks a rotation mid-verify, to prove the rescue block restores access molecule test -s rollback ``` +If you touch anything that writes or restores `authorized_keys`, run `nonroot` and `rollback`. +They rotate a non-root user, which is the only way an ownership mistake shows up: the role works +under `become`, and a file left owned by root is silently ignored by sshd's `StrictModes`, locking +the account out. With a `root` target that same bug looks correct. + Both scenarios generate their own ephemeral SSH keypairs and inventory into Molecule's per-run directory, so nothing needs to exist in your working tree for them to run. If you are iterating and do not want the container torn down between runs: diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index c3a222f..32275de 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -24,11 +24,13 @@ krameff-ssh_key_rotation/ │ ├── ISSUE_TEMPLATE/ # Bug and feature forms; config.yml routes security reports privately │ └── workflows/ │ ├── ci.yml # Lint + syntax-check on push/PR -│ ├── molecule.yml # Functional Molecule tests (default + rollback) on push/PR, plus a weekly run +│ ├── molecule.yml # Functional Molecule tests (all 3 scenarios) on push/PR, plus a weekly run │ └── release.yml # Builds and publishes to Galaxy on a v* tag ├── extensions/ │ └── molecule/ # Molecule scenarios (the location current Molecule expects) │ ├── default/ # Full rotation over real SSH, twice, to also prove idempotency +│ ├── nonroot/ # Same, rotating a non-root user, to catch file-ownership mistakes +│ ├── resources/ # Shared prepare playbook and Dockerfile used by all scenarios │ └── rollback/ # Breaks a rotation mid-verify to prove the rescue block restores access ├── meta/ │ └── runtime.yml # Ansible version requirements @@ -41,6 +43,7 @@ krameff-ssh_key_rotation/ │ ├── meta/main.yml # Role metadata (platforms, min Ansible version) │ └── tasks/ │ ├── main.yml # Fails fast; this role has no default entry point, see below +│ ├── resolve_sshd_anchor.yml # Works out where the managed sshd_config blocks belong │ ├── validate.yml # Phase 0: local pre-flight validation │ ├── install.yml # Phase 1: install the new key, prepare sshd (connect via OLD key) │ ├── verify.yml # Phase 2: verify the new key, then remove the old key/legacy auth @@ -137,18 +140,28 @@ ansible-playbook -i inventory.ini playbooks/rotate.yml --check ### Molecule -Two scenarios live under `extensions/molecule/`. They run against live containers (Ubuntu 22.04, Rocky Linux 9, Rocky Linux 10) with sshd installed and running. +Three scenarios live under `extensions/molecule/`. They run against live containers (Ubuntu 22.04, Rocky Linux 9, Rocky Linux 10) with sshd installed and running. ```bash # Full rotation via playbooks/rotate.yml, run twice to also prove idempotency molecule test -s default +# The same rotation, but against a non-root user whose primary group differs from its name +molecule test -s nonroot + # Runs the role's stages directly, breaking the second rotation mid-verify to prove the # block/rescue in roles/ssh_key_rotation/tasks/verify.yml actually restores access molecule test -s rollback ``` -Both scenarios generate their own ephemeral SSH keypairs and inventory into Molecule's per-run directory, so nothing needs to exist in the repository for them to run. +All three generate their own ephemeral SSH keypairs and inventory into Molecule's per-run directory, so nothing needs to exist in the repository for them to run. + +`nonroot` and `rollback` rotate a non-root account on purpose. The role does its file work under +`become`, so an `authorized_keys` it writes for a non-root user has to be chowned back explicitly; +if it is not, sshd's `StrictModes` ignores the file and refuses every key. That failure is +invisible when the target is `root`, because root-owned is then the correct result, so a root-only +suite cannot catch it. `rollback` is the important one, since the bug's natural home is the +`rescue` path. While iterating, avoid the full create-and-destroy cycle each time: @@ -213,6 +226,11 @@ Keep generated keys out of the working tree. `.gitignore` covers the common patt The collection follows [Semantic Versioning](https://semver.org/): +The collection is pre-1.0 and has not been released, so there is nothing published to stay +compatible with. Until 1.0.0 ships, behaviour may change within `0.x` releases without a major +bump; the changes are still written up in `CHANGELOG.md` so they are never a surprise. The scheme +below applies from 1.0.0 onwards. + - **Major** (1.0.0 to 2.0.0), for breaking changes: renamed or retyped variables, removed playbooks or features, or changed behaviour that could affect an existing workflow. - **Minor** (1.0.0 to 1.1.0), for backwards-compatible features: new playbooks or roles, new variables with sensible defaults, new optional capabilities. - **Patch** (1.0.0 to 1.0.1), for backwards-compatible fixes: configuration fixes, documentation fixes, small improvements that do not change behaviour. @@ -225,7 +243,7 @@ New variables should default to off or empty, so that upgrading never changes wh ```bash ansible-galaxy collection build . -# Output: ./krameff-ssh_key_rotation-1.0.0.tar.gz +# Output: ./krameff-ssh_key_rotation-0.9.0.tar.gz ``` `galaxy.yml`'s `build_ignore` keeps the Molecule scenarios, CI configuration, editor and agent directories, and any local inventory or key material out of the tarball. @@ -239,7 +257,7 @@ So the order is: bump `version` in `galaxy.yml`, move the `Unreleased` changelog To publish by hand if you need to: ```bash -ansible-galaxy collection publish ./krameff-ssh_key_rotation-1.0.0.tar.gz \ +ansible-galaxy collection publish ./krameff-ssh_key_rotation-0.9.0.tar.gz \ --api-key ``` diff --git a/PQC.md b/PQC.md new file mode 100644 index 0000000..2343312 --- /dev/null +++ b/PQC.md @@ -0,0 +1,122 @@ +# Post-quantum algorithm negotiation + +Opt-in support for post-quantum and hybrid SSH algorithms: what the collection can manage, where +each piece runs, and how it interacts with RHEL/Fedora crypto-policies. + +Everything here is off by default. If you are not deliberately enabling PQC, you can ignore this +page entirely. Start at [README.md](README.md), or [QUICKSTART.md](QUICKSTART.md) for a first rotation. + +`ssh_key_rotation_pqc_key_types` only affects local key *type* recognition in Phase 0. It never changes what `sshd` and `ssh` actually negotiate on the wire. + +For a PQC or hybrid key to work end to end, both sides of the connection need matching algorithm lists across several independent layers. This collection can manage all of them: + +| Variable | What it affects | +|----------|-----------------| +| `ssh_key_rotation_pqc_kex_algorithms` | `KexAlgorithms` in the target's `sshd_config`, and `-o KexAlgorithms` for this playbook's own connections | +| `ssh_key_rotation_pqc_pubkey_algorithms` | `PubkeyAcceptedAlgorithms` and `HostKeyAlgorithms` in the target's `sshd_config`, and `-o PubkeyAcceptedAlgorithms` for this playbook's own connections | +| `ssh_key_rotation_pqc_ca_signature_algorithms` | `CASignatureAlgorithms` in the target's `sshd_config`, for SSH CA setups only | +| `ssh_key_rotation_manage_crypto_policy` | Whether the RHEL/Fedora system-wide crypto-policy is also managed, on the control node and on targets | +| `ssh_key_rotation_crypto_policy_setting` | The value passed to `update-crypto-policies --set`, optionally combining a base policy with subpolicy modules using `BASE:MODULE` syntax, such as `DEFAULT:PQ` or `FIPS:PQ` | +| `ssh_key_rotation_crypto_policy_add_modules` | Preferred over the setting above: adds modules onto a host's current policy instead of replacing it | + +Every `sshd_config` directive is written with OpenSSH's `+algorithm` syntax, appending to the compiled-in defaults rather than replacing the list outright, so clients that do not speak PQC yet can still fall back to a classical algorithm. + +## Where each piece runs + +The diagram below splits the work by stage and by machine. See [Role reference](README.md#role-reference) for how these map to `tasks/*.yml`. + +```mermaid +flowchart TD + subgraph P0["validate stage - control node, before any host is touched"] + Vars{"Any PQC algorithms requested?"} + Vars -->|"No"| Skip0["Nothing to do - skip straight to the install stage"] + Vars -->|"Yes"| ControlCheck{"Does the control node's own ssh support them?"} + ControlCheck -->|"No"| WarnLocal["Warn: negotiation may fail later"] + ControlCheck -->|"Yes"| ControlPolicy{"Also manage the control node's crypto-policy?"} + WarnLocal --> ControlPolicy + ControlPolicy -->|"Yes"| ControlModuleCheck{"Does the policy need a subpolicy module, e.g. FIPS:PQ?"} + ControlModuleCheck -->|"Yes"| ControlModuleGate["Fail now if that module isn't installed"] + ControlModuleCheck -->|"No"| ControlApply["Apply the crypto-policy"] + ControlModuleGate --> ControlApply + ControlPolicy -->|"No"| ExtraArgs + ControlApply --> ExtraArgs["Carry the algorithms into this playbook's own ssh connections"] + end + + subgraph P1["install stage - target host, connected with the OLD key"] + Record["Record pre-install state in /etc/ansible/facts.d"] + Record --> WriteDropin["Write the algorithms to this role's drop-in + (sshd_config itself is never edited)"] + WriteDropin --> TargetPolicy{"Also manage the target's crypto-policy?"} + TargetPolicy -->|"Yes"| TargetModuleCheck{"Does the policy need a subpolicy module?"} + TargetModuleCheck -->|"Yes"| TargetModuleGate["Fail now if that module isn't installed"] + TargetModuleCheck -->|"No"| TargetApply["Apply the crypto-policy"] + TargetModuleGate --> TargetApply + TargetApply --> Validate["Validate the MERGED sshd config"] + TargetPolicy -->|"No"| Validate + Validate --> Reload["Reload sshd"] + Reload --> EffectiveCheck{"Did the algorithms actually take effect, + and will sshd accept the new key's type?"} + EffectiveCheck -->|"No"| InstallRollback["Roll back: remove this role's drop-in, + restore authorized_keys, restore the crypto-policy"] + EffectiveCheck -->|"Yes"| Reconnect + end + + subgraph P2["verify stage - target host, reconnecting with the NEW key"] + Reconnect{"Does the new key authenticate over the new algorithms?"} + Reconnect -->|"No"| AbortVerify["Abort - old key and legacy auth left untouched"] + Reconnect -->|"Yes"| RemoveOldKey["Back up, then remove the OLD key"] + RemoveOldKey --> Cleanup["Optionally disable password/keyboard-interactive auth + via a second drop-in"] + end + + ExtraArgs --> Record +``` + +**On the control node, in Phase 0.** `ssh -Q kex` and `ssh -Q key-sig` confirm your own ssh binary can offer the algorithms you are asking for, before any host is touched. If `ssh_key_rotation_manage_crypto_policy` is set and this is a RHEL or Fedora control node, `update-crypto-policies --set` runs there too, so the machine's ssh *client* backend permits PQC algorithms system-wide. This is detected by checking whether the `update-crypto-policies` tool exists, not by an OS-family fact. + +**On the connection itself.** The algorithms need to travel with the Ansible connection. Rather than editing any file on the control node, Phases 1 and 2 compute an `ansible_ssh_extra_args` value that passes `-o KexAlgorithms=+...` and `-o PubkeyAcceptedAlgorithms=+...` for this playbook's connections only. + +**On the target, in Phase 1.** `KexAlgorithms`, `PubkeyAcceptedAlgorithms`, `HostKeyAlgorithms` and `CASignatureAlgorithms` are appended to `sshd_config`, using the same `lineinfile` plus `sshd -t` plus backup pattern used everywhere else. + +If `ssh_key_rotation_manage_crypto_policy` is set and the target has the tooling, the crypto-policy step runs there too. Because `update-crypto-policies --set` validates only its own module syntax and not the resulting merged `sshd_config`, there is an explicit `sshd -t` re-check afterwards, before the reload handler is allowed to fire. The drop-in warning also calls out `50-redhat.conf` by name, since that file is the generated crypto-policy backend include and is not meant to be hand-edited. + +Once `sshd` has reloaded, the `sshd -T` check flags any requested algorithm that still is not showing up in the effective config, so you find out before Phase 2 tries and fails to reconnect. + +## Combining a base policy with a subpolicy module + +`update-crypto-policies --set` accepts either a base policy name on its own (`DEFAULT`, `FIPS`, `LEGACY` and so on) or a base policy combined with one or more subpolicy *modules*, written as `BASE:MODULE`. For example `FIPS:PQ`, or `FIPS:PQ:NO-SHA1` to stack more than one. + +Each module is a `MODULE.pmod` file, either shipped by the OS under `/usr/share/crypto-policies/policies/modules/` or dropped in locally under `/etc/crypto-policies/policies/modules/`. A module is added on top of the base policy rather than replacing it, so `FIPS:PQ` stays FIPS-compliant everywhere else and only adds what `PQ.pmod` grants. + +This matters specifically for PQC. On AlmaLinux and RHEL 9, the `FIPS` policy alone includes no post-quantum key-exchange groups, but the OS still ships a built-in `PQ.pmod` that adds `mlkem768x25519-sha256` and other ML-KEM groups when combined as `FIPS:PQ`. This was confirmed against a real AlmaLinux 9.8 host, where `sshd -T` only showed `mlkem768x25519-sha256` in the effective `KexAlgorithms` after switching from `FIPS` to `FIPS:PQ`. AlmaLinux and RHEL 10 ship PQC key exchange in `FIPS` already, so the combination is not needed there. + +Before ever calling `update-crypto-policies --set`, the playbook lists whatever `*.pmod` files exist under both module directories on that host, the control node in Phase 0 and the target in Phase 1. If `ssh_key_rotation_crypto_policy_setting` names a module that is not present, it fails before making any change, rather than letting `update-crypto-policies` silently ignore an unknown module name or fail in a way that is easy to miss in the task output. + +## Examples + +All three run the same command as [Usage](README.md#usage); only `rotation_vars.yml` differs. + +**Enable a PQC key-exchange algorithm:** + +```yaml +ssh_key_rotation_pqc_kex_algorithms: + - mlkem768x25519-sha256 +``` + +**Manage the RHEL/Fedora crypto-policy as well:** + +```yaml +ssh_key_rotation_manage_crypto_policy: true +ssh_key_rotation_crypto_policy_setting: "DEFAULT:PQ" +``` + +**Add PQC to a FIPS-mode host,** keeping the rest of FIPS intact. Note the new key must be +ECDSA or RSA here, since FIPS does not accept ed25519: + +```yaml +new_private_key: "./pwc_id_ecdsa" +new_public_key_file: "./pwc_id_ecdsa.pub" +ssh_key_rotation_manage_crypto_policy: true +ssh_key_rotation_crypto_policy_add_modules: + - PQ +``` diff --git a/QUICKSTART.md b/QUICKSTART.md index a3d21ac..c496156 100644 --- a/QUICKSTART.md +++ b/QUICKSTART.md @@ -1,120 +1,112 @@ -# Quick Start Guide +# Quick start -Get SSH key rotation running in 5 minutes. +Rotate your first key in five minutes. For the full detail, see [README.md](README.md). -## 1. Install the Collection +> **Try it on a disposable host first.** A VM you can snapshot and roll back is ideal. The +> playbook is built not to lock you out, but SSH lockouts are painful enough to be worth the +> dry run. + +## 1. Install ```bash -# From Ansible Galaxy ansible-galaxy collection install krameff.ssh_key_rotation - -# Or install dependencies first -ansible-galaxy install -r requirements.yml +ansible-galaxy collection install ansible.posix ``` -## 2. Generate Your Keys - -If you don't already have keys: +## 2. Generate the new key ```bash -# Generate new ed25519 key pair (recommended) -ssh-keygen -t ed25519 -f pwc_id_ed25519 -C "your-email@example.com" - -# Or use RSA (older but widely supported) -ssh-keygen -t rsa -b 4096 -f id_rsa_new -C "your-email@example.com" +ssh-keygen -t ed25519 -f ./pwc_id_ed25519 -C "your-email@example.com" ``` -## 3. Create Your Inventory +On a RHEL-family host in FIPS mode, use `-t ecdsa -b 521` instead: the FIPS crypto-policy does +not accept ed25519 keys. The playbook checks this for you and stops safely if it is wrong. + +## 3. List your hosts -Copy and customize the example: +Copy the example and edit it. The playbook rotates the `rotate` group. ```bash cp inventory.example.ini inventory.ini ``` -Edit `inventory.ini` with your target hosts: - ```ini [rotate] prod-web-01 ansible_host=10.0.1.10 ansible_user=ubuntu -prod-db-01 ansible_host=10.0.2.10 ansible_user=ec2-user +prod-db-01 ansible_host=10.0.2.10 ansible_user=ec2-user ``` -## 4. Run the Playbook +## 4. Point at your keys + +Copy the example and fill in the four paths. They are files on **your** machine, not the targets. ```bash -ansible-playbook krameff.ssh_key_rotation.rotate \ - -i inventory.ini \ - -e "old_private_key=~/.ssh/id_rsa" \ - -e "new_private_key=./pwc_id_ed25519" \ - -e "new_public_key_file=./pwc_id_ed25519.pub" \ - -e "old_public_key_file=~/.ssh/id_rsa.pub" \ - --ask-become-pass +cp rotation_vars.example.yml rotation_vars.yml ``` -**What happens:** -- Phase 1: Installs new key on all hosts, enables pubkey auth -- Phase 2: Reconnects with new key, verifies it works, removes old key - -## 5. Verify Success +```yaml +old_private_key: "~/.ssh/id_old" # the key you can log in with today +old_public_key_file: "~/.ssh/id_old.pub" +new_private_key: "./pwc_id_ed25519" # the key you want to end up with +new_public_key_file: "./pwc_id_ed25519.pub" +``` -After the playbook completes, verify you can still SSH: +## 5. Rotate ```bash -ssh -i ./pwc_id_ed25519 ubuntu@10.0.1.10 +ansible-playbook krameff.ssh_key_rotation.rotate \ + -i inventory.ini -e @rotation_vars.yml --ask-become-pass ``` -If successful, you can delete the old private key: +What happens, in order: -```bash -rm ~/.ssh/id_rsa -``` +1. **Validate**, on your machine only. Checks the key paths, type and strength before any host is touched. +2. **Install**, over the *old* key. Adds the new key alongside it and enables public key auth. +3. **Verify**, over the *new* key. Proves the new key works, and only then removes the old key and disables password login. -## Common Options +The old key is never removed until the new one has authenticated. If anything fails partway +through the cleanup, the playbook restores what it changed and tells you exactly what it put back. -### Disable specific auth methods only +## 6. Confirm ```bash -ansible-playbook krameff.ssh_key_rotation.rotate \ - -i inventory.ini \ - -e old_private_key=~/.ssh/id_rsa \ - -e new_private_key=./pwc_id_ed25519 \ - -e new_public_key_file=./pwc_id_ed25519.pub \ - -e old_public_key_file=~/.ssh/id_rsa.pub \ - -e ssh_key_rotation_disable_password_auth=true \ - -e ssh_key_rotation_disable_kbd_interactive=false +ssh -i ./pwc_id_ed25519 ubuntu@10.0.1.10 ``` -### Keep multiple keys active +Once that works, the old private key can go: ```bash -ansible-playbook krameff.ssh_key_rotation.rotate \ - -i inventory.ini \ - -e old_private_key=~/.ssh/id_rsa \ - -e new_private_key=./pwc_id_ed25519 \ - -e new_public_key_file=./pwc_id_ed25519.pub \ - -e old_public_key_file=~/.ssh/id_rsa.pub \ - -e ssh_key_rotation_make_exclusive=false +rm ~/.ssh/id_old ``` -## Troubleshooting +## Common tweaks + +Add these to `rotation_vars.yml`: + +```yaml +ssh_key_rotation_disable_password_auth: false # leave password login enabled +ssh_key_rotation_disable_kbd_interactive: false # leave keyboard-interactive enabled +ssh_key_rotation_make_exclusive: true # leave ONLY the new key in authorized_keys +ssh_key_rotation_target_user: "ubuntu" # rotate a specific account +``` -**"New key did not authenticate"** - Phase 2 cannot connect -- Verify the new key paths are correct -- Check that Phase 1 completed without errors -- Confirm the new key is in `~/.ssh/authorized_keys` on the target +## If something goes wrong -**"Permission denied"** during Phase 1 -- Make sure the old private key is correct and readable -- Check that `ansible_user` matches the remote user -- Verify SSH is not blocked by firewall rules +Nothing below leaves a host half-changed. The playbook either completes or puts things back. -**Want more help?** → See [README.md](README.md) +| Message | What it means | +|---------|---------------| +| `Missing required variables` | One of the four paths is unset. Nothing has run yet. | +| `does not match new_public_key_file` | Your new private and public keys are not a pair. Check both paths. | +| `crypto-policy would reject the new key entirely` | The host will not accept this key type, commonly ed25519 under FIPS. Generate an ECDSA or RSA key. Stopped before touching anything. | +| `New key did not authenticate` | The new key could not log in, so the old key was left alone. Check it reached `authorized_keys`. | +| `still reports password authentication as ENABLED` | A file in `/etc/ssh/sshd_config.d/` is overriding the lock-down. Fix it there, or set `ssh_key_rotation_manage_sshd_dropin: true`, and re-run. Changes were rolled back. | +| `STILL ENABLED for ` | A `Match` block re-enables password login for the account you rotated. The role never edits `Match` blocks; amend it by hand and re-run. Changes were rolled back. | +| `Permission denied` during install | The old key or `ansible_user` is wrong. Re-run with `-vvv`. | ---- +Still stuck? See [Troubleshooting](README.md#troubleshooting) in the README. -## Next Steps +## Next steps -- Review [README.md](README.md) for detailed documentation -- Check [CHANGELOG.md](CHANGELOG.md) for version history -- Run with `-vvv` flag for verbose output during testing +- [README.md](README.md) for variables, the safety model and post-quantum options +- [CHANGELOG.md](CHANGELOG.md) for what has changed diff --git a/README.md b/README.md index 832faf5..55f5a1a 100644 --- a/README.md +++ b/README.md @@ -23,8 +23,10 @@ An Ansible collection for rotating SSH keys across your infrastructure without b - [Usage](#usage) - [Variables](#variables) - [Key validation](#key-validation) -- [PQC algorithm negotiation](#pqc-algorithm-negotiation) +- [Drop-in overrides](#drop-in-overrides) +- [Post-quantum algorithms](#post-quantum-algorithms) (full detail in [PQC.md](PQC.md)) - [Safety model](#safety-model) +- [Limitations](#limitations) - [What each phase does](#what-each-phase-does) - [Role reference](#role-reference) - [Troubleshooting](#troubleshooting) @@ -54,13 +56,15 @@ Every `sshd_config` change is checked with `sshd -t` before it is written, and b **Cross-OS support.** The Debian/Ubuntu vs RHEL/CentOS `sshd` service-name difference is handled for you. -**Drop-in and `Match` block awareness.** Warns if files in `/etc/ssh/sshd_config.d/` or `Match` blocks might silently override what the playbook is setting. +**Lock-down is verified, not assumed.** After disabling password and keyboard-interactive auth, the playbook re-reads `sshd -T` and fails if either is still enabled, so a drop-in file cannot quietly leave password login working on a run that reported success. + +**Drop-in and `Match` block awareness.** Warns about files in `/etc/ssh/sshd_config.d/` that could override the main config, and verifies the lock-down applies to the rotated user specifically, so a `Match User` block re-enabling password login is caught rather than missed. Optionally writes its own drop-in so the lock-down wins outright. See [Drop-in overrides](#drop-in-overrides). **Flexible auth policy.** Optionally make the new key exclusive, and optionally disable password and keyboard-interactive auth. **Validated, backed-up config edits.** Every `sshd` change is checked with `sshd -t` and backed up before it is written. -**Post-quantum negotiation, opt-in.** Can enable post-quantum and hybrid key exchange and signature algorithms across `sshd_config`, RHEL/Fedora crypto-policy, and the control node's own ssh client. See [PQC algorithm negotiation](#pqc-algorithm-negotiation). +**Post-quantum negotiation, opt-in.** Can enable post-quantum and hybrid key exchange and signature algorithms across `sshd_config`, RHEL/Fedora crypto-policy, and the control node's own ssh client. See [PQC.md](PQC.md). ## Requirements @@ -78,10 +82,15 @@ The collection targets Debian/Ubuntu and RHEL/Fedora-family hosts generally. The | OS | Notes | |----|-------| | Ubuntu 22.04 LTS (Jammy Jellyfish) | Watch for `sshd_config.d/` drop-ins such as cloud-init's `50-cloud-init.conf`, which can override settings written further down `sshd_config`. See [Drop-in sshd config files found](#drop-in-sshd-config-files-found). | -| AlmaLinux 9.8 (Olive Jaguar), `FIPS` crypto-policy | `FIPS` on its own has no PQC key exchange. Use `ssh_key_rotation_crypto_policy_add_modules: [PQ]` to add it. See [Combining a base policy with a subpolicy module](#combining-a-base-policy-with-a-subpolicy-module). | -| AlmaLinux 10.2 (Lavender Lion), `FIPS` crypto-policy | `FIPS` already includes PQC key exchange here. No extra module needed. | +| Ubuntu 26.04 LTS, OpenSSH 10.2 | Ships `50-cloud-init.conf` with `PasswordAuthentication yes`, which overrides the lock-down. Phase 2 fails rather than reporting a success it did not achieve. Fix the drop-in and re-run. | +| Rocky Linux 9.8 (Blue Onyx), `FIPS` crypto-policy | `FIPS` excludes `ssh-ed25519`, so ed25519 keys are rejected. Use ECDSA or RSA. `FIPS` alone has no PQC key exchange; add it with `ssh_key_rotation_crypto_policy_add_modules: [PQ]`. | +| AlmaLinux 9.8 (Olive Jaguar), `FIPS` crypto-policy | `FIPS` on its own has no PQC key exchange. Use `ssh_key_rotation_crypto_policy_add_modules: [PQ]` to add it. See [Combining a base policy with a subpolicy module](PQC.md#combining-a-base-policy-with-a-subpolicy-module). | +| AlmaLinux 10.1 / 10.2 (Lavender Lion), `FIPS` crypto-policy | `FIPS` already includes PQC key exchange here. No extra module needed. As on 9.x, ed25519 is not accepted under FIPS. | | openSUSE Leap 15.6 | Ships Python 3.6 by default, which is too old for Ansible 2.15+ on the target side. Set `ansible_python_interpreter` to a Python 3.7+ install, for example `python39` via `zypper`. | +Container-based tests (`molecule`) additionally cover Ubuntu 22.04, Rocky Linux 9 and Rocky Linux 10, +rotating both `root` and a non-root user. + Other versions in the same families are likely to work, since nothing here relies on version-specific behaviour beyond what is called out above. They have not been explicitly verified. ## Installation @@ -91,22 +100,14 @@ Other versions in the same families are likely to work, since nothing here relie ansible-galaxy collection install krameff.ssh_key_rotation # From a tarball or directory -ansible-galaxy collection install /path/to/krameff-ssh_key_rotation-1.0.0.tar.gz +ansible-galaxy collection install /path/to/krameff-ssh_key_rotation-0.9.0.tar.gz ``` ## Usage -### Basic example +New to this? [QUICKSTART.md](QUICKSTART.md) walks through a first rotation end to end. -```bash -ansible-playbook krameff.ssh_key_rotation.rotate \ - -i inventory.ini \ - -e "old_private_key=~/.ssh/id_old" \ - -e "new_private_key=~/.ssh/pwc_id_ed25519" \ - -e "new_public_key_file=./pwc_id_ed25519.pub" \ - -e "old_public_key_file=./id_old.pub" \ - --ask-become-pass -``` +### Set up once The playbook runs against the `rotate` host group, which you define in your inventory: @@ -117,36 +118,59 @@ prod-web-02 ansible_host=10.0.1.11 prod-db-01 ansible_host=10.0.2.10 ``` -### Disable password auth but keep keyboard-interactive +Every rotation needs four paths, all on the control node. Keep them in a vars file rather than +retyping four `-e` flags each time: ```bash -ansible-playbook krameff.ssh_key_rotation.rotate \ - -i inventory.ini \ - -e old_private_key=~/.ssh/id_old \ - -e new_private_key=~/.ssh/pwc_id_ed25519 \ - -e new_public_key_file=./pwc_id_ed25519.pub \ - -e old_public_key_file=./id_old.pub \ - -e ssh_key_rotation_disable_password_auth=true \ - -e ssh_key_rotation_disable_kbd_interactive=false +cp rotation_vars.example.yml rotation_vars.yml # then edit the four paths ``` -### Keep the old key in place as well as the new one +```yaml +old_private_key: "~/.ssh/id_old" +old_public_key_file: "~/.ssh/id_old.pub" +new_private_key: "./pwc_id_ed25519" +new_public_key_file: "./pwc_id_ed25519.pub" +``` + +Your copy is gitignored. Everything in [Variables](#variables) can go in the same file. + +### Run it ```bash ansible-playbook krameff.ssh_key_rotation.rotate \ - -i inventory.ini \ - -e old_private_key=~/.ssh/id_old \ - -e new_private_key=~/.ssh/pwc_id_ed25519 \ - -e new_public_key_file=./pwc_id_ed25519.pub \ - -e old_public_key_file=./id_old.pub \ - -e ssh_key_rotation_make_exclusive=false \ - -e ssh_key_rotation_disable_password_auth=true + -i inventory.ini -e @rotation_vars.yml --ask-become-pass +``` + +The four paths can still be passed as `-e old_private_key=...` if you prefer; the vars file is +only a convenience. + +### Common variations + +Each of these is a line to add to `rotation_vars.yml`. Nothing else about the command changes. + +```yaml +# Disable password auth but keep keyboard-interactive +ssh_key_rotation_disable_password_auth: true +ssh_key_rotation_disable_kbd_interactive: false + +# Keep the old key in place alongside the new one +ssh_key_rotation_make_exclusive: false + +# Leave ONLY the new key in authorized_keys +ssh_key_rotation_make_exclusive: true + +# Rotate a specific account rather than the inventory's ansible_user +ssh_key_rotation_target_user: "ubuntu" ``` ## Variables +Any of these can go in `rotation_vars.yml` (see [Usage](#usage)) or be passed with `-e`. + ### Required +All four are paths on the **control node**, not on the targets. + | Variable | Type | Description | |----------|------|-------------| | `old_private_key` | string | Path to the old SSH private key, on the control node | @@ -154,6 +178,10 @@ ansible-playbook krameff.ssh_key_rotation.rotate \ | `old_public_key_file` | string | Path to the old SSH public key, on the control node | | `new_public_key_file` | string | Path to the new SSH public key, on the control node | +These have no defaults, deliberately. A default in `defaults/main.yml` would count as "defined" +and permanently disable the Phase 0 check that catches a missing or misspelled path before any +host is touched. Copy `rotation_vars.example.yml` instead. + ### Optional | Variable | Type | Default | Description | @@ -162,6 +190,11 @@ ansible-playbook krameff.ssh_key_rotation.rotate \ | `ssh_key_rotation_disable_password_auth` | bool | `true` | Disable password authentication after rotation | | `ssh_key_rotation_disable_kbd_interactive` | bool | `true` | Disable keyboard-interactive auth after rotation | | `ssh_key_rotation_make_exclusive` | bool | `false` | Leave only the new key in `authorized_keys` | +| `ssh_key_rotation_sshd_dropin_prefix` | string | `"99"` | Numeric prefix for this role's drop-ins. Lower wins. See [Drop-in overrides](#drop-in-overrides) | +| `ssh_key_rotation_sshd_dropin_dir` | string | `/etc/ssh/sshd_config.d` | Where this role writes its configuration | +| `ssh_key_rotation_manage_sshd_dropin` | bool | `false` | Deprecated: equivalent to a prefix of `01`, making this role outrank administrator drop-ins | +| `ssh_key_rotation_rollback_remove_new_key` | bool | `false` | On rollback, also restore the pre-install `authorized_keys`, removing the new key | +| `ssh_key_rotation_check_match_blocks` | bool | `true` | Check the lock-down applies to the rotated user specifically, catching a `Match` block that re-enables password auth for them | | `ssh_key_rotation_accepted_key_types` | list | `[ED25519, ED25519-SK, ECDSA, ECDSA-SK, RSA]` | Key types allowed for `new_public_key_file` | | `ssh_key_rotation_reject_key_types` | list | `[DSA]` | Key types that always fail validation, whatever the accepted list says | | `ssh_key_rotation_min_rsa_bits` | int | `3072` | Minimum RSA key size accepted | @@ -169,7 +202,7 @@ ansible-playbook krameff.ssh_key_rotation.rotate \ ### Optional, post-quantum -All of these are empty or false by default, so none of them change existing behaviour unless you ask for them. See [PQC algorithm negotiation](#pqc-algorithm-negotiation). +All of these are empty or false by default, so none of them change existing behaviour unless you ask for them. See [PQC.md](PQC.md). | Variable | Type | Default | Description | |----------|------|---------|-------------| @@ -204,128 +237,98 @@ This matters because a system-wide crypto-policy can quietly narrow what the Pha Without this check, Phase 1 would report success and Phase 2 would go on to remove the old key, leaving the host with no key it can actually authenticate with. The check runs, and can fail the host, before Phase 2 does anything destructive. -## PQC algorithm negotiation +## Post-quantum algorithms -`ssh_key_rotation_pqc_key_types` only affects local key *type* recognition in Phase 0. It never changes what `sshd` and `ssh` actually negotiate on the wire. +Opt-in, and off by default. The collection can enable post-quantum and hybrid key exchange and +signature algorithms across the target's `sshd_config`, the RHEL/Fedora system-wide crypto-policy, +and the control node's own ssh client. -For a PQC or hybrid key to work end to end, both sides of the connection need matching algorithm lists across several independent layers. This collection can manage all of them: +If you need it, it is all on one page: **[PQC.md](PQC.md)**. It covers the variables, where each +piece runs across the three phases, and how `FIPS:PQ`-style crypto-policy modules work. -| Variable | What it affects | -|----------|-----------------| -| `ssh_key_rotation_pqc_kex_algorithms` | `KexAlgorithms` in the target's `sshd_config`, and `-o KexAlgorithms` for this playbook's own connections | -| `ssh_key_rotation_pqc_pubkey_algorithms` | `PubkeyAcceptedAlgorithms` and `HostKeyAlgorithms` in the target's `sshd_config`, and `-o PubkeyAcceptedAlgorithms` for this playbook's own connections | -| `ssh_key_rotation_pqc_ca_signature_algorithms` | `CASignatureAlgorithms` in the target's `sshd_config`, for SSH CA setups only | -| `ssh_key_rotation_manage_crypto_policy` | Whether the RHEL/Fedora system-wide crypto-policy is also managed, on the control node and on targets | -| `ssh_key_rotation_crypto_policy_setting` | The value passed to `update-crypto-policies --set`, optionally combining a base policy with subpolicy modules using `BASE:MODULE` syntax, such as `DEFAULT:PQ` or `FIPS:PQ` | -| `ssh_key_rotation_crypto_policy_add_modules` | Preferred over the setting above: adds modules onto a host's current policy instead of replacing it | +Note that `ssh_key_rotation_pqc_key_types` only affects local key *type* recognition in Phase 0. +It does not change what `sshd` and `ssh` actually negotiate on the wire; that needs the algorithm +variables described in PQC.md. -Every `sshd_config` directive is written with OpenSSH's `+algorithm` syntax, appending to the compiled-in defaults rather than replacing the list outright, so clients that do not speak PQC yet can still fall back to a classical algorithm. +## Drop-in overrides -### Where each piece runs +Worth understanding before you rely on the lock-down, because the rule is the opposite of what +most people assume. -The diagram below splits the work by stage and by machine. See [Role reference](#role-reference) for how these map to `tasks/*.yml`. +Every mainstream distribution now starts `/etc/ssh/sshd_config` with: -```mermaid -flowchart TD - subgraph P0["validate stage - control node, before any host is touched"] - Vars{"Any PQC algorithms requested?"} - Vars -->|"No"| Skip0["Nothing to do - skip straight to the install stage"] - Vars -->|"Yes"| ControlCheck{"Does the control node's own ssh support them?"} - ControlCheck -->|"No"| WarnLocal["Warn: negotiation may fail later"] - ControlCheck -->|"Yes"| ControlPolicy{"Also manage the control node's crypto-policy?"} - WarnLocal --> ControlPolicy - ControlPolicy -->|"Yes"| ControlModuleCheck{"Does the policy need a subpolicy module, e.g. FIPS:PQ?"} - ControlModuleCheck -->|"Yes"| ControlModuleGate["Fail now if that module isn't installed"] - ControlModuleCheck -->|"No"| ControlApply["Apply the crypto-policy"] - ControlModuleGate --> ControlApply - ControlPolicy -->|"No"| ExtraArgs - ControlApply --> ExtraArgs["Carry the algorithms into this playbook's own ssh connections"] - end - - subgraph P1["install stage - target host, connected with the OLD key"] - SshdConfig["Add the algorithms to the target's sshd_config"] - SshdConfig --> TargetPolicy{"Also manage the target's crypto-policy?"} - TargetPolicy -->|"Yes"| TargetModuleCheck{"Does the policy need a subpolicy module?"} - TargetModuleCheck -->|"Yes"| TargetModuleGate["Fail now if that module isn't installed"] - TargetModuleCheck -->|"No"| TargetApply["Apply the crypto-policy"] - TargetModuleGate --> TargetApply - TargetApply --> RevalidateConfig["Re-check sshd_config is still valid"] - TargetPolicy -->|"No"| DropinCheck - RevalidateConfig --> DropinCheck["Warn about drop-in files/Match blocks that could override this"] - DropinCheck --> Reload["Reload sshd"] - Reload --> EffectiveCheck["Confirm the algorithms actually took effect"] - end - - subgraph P2["verify stage - target host, reconnecting with the NEW key"] - Reconnect{"Does the new key authenticate over the new algorithms?"} - Reconnect -->|"No"| AbortVerify["Abort - old key and legacy auth left untouched"] - Reconnect -->|"Yes"| BackupVerify["Back up authorized_keys again"] - BackupVerify --> RemoveOldKey["Remove the OLD key from authorized_keys"] - RemoveOldKey --> Cleanup["Optionally disable password/keyboard-interactive auth"] - end - - ExtraArgs --> SshdConfig - EffectiveCheck --> Reconnect +``` +Include /etc/ssh/sshd_config.d/*.conf ``` -**On the control node, in Phase 0.** `ssh -Q kex` and `ssh -Q key-sig` confirm your own ssh binary can offer the algorithms you are asking for, before any host is touched. If `ssh_key_rotation_manage_crypto_policy` is set and this is a RHEL or Fedora control node, `update-crypto-policies --set` runs there too, so the machine's ssh *client* backend permits PQC algorithms system-wide. This is detected by checking whether the `update-crypto-policies` tool exists, not by an OS-family fact. - -**On the connection itself.** The algorithms need to travel with the Ansible connection. Rather than editing any file on the control node, Phases 1 and 2 compute an `ansible_ssh_extra_args` value that passes `-o KexAlgorithms=+...` and `-o PubkeyAcceptedAlgorithms=+...` for this playbook's connections only. +and **sshd keeps the first value it sees for a keyword**. Not the last. Because the `Include` +is at the top, a drop-in beats anything written further down the main file, and among drop-ins +the lowest-sorting filename wins. Verified on Ubuntu 26.04: against a `50-cloud-init.conf` +containing `PasswordAuthentication yes`, a `99-*.conf` setting it to `no` has no effect, while +an `01-*.conf` does. -**On the target, in Phase 1.** `KexAlgorithms`, `PubkeyAcceptedAlgorithms`, `HostKeyAlgorithms` and `CASignatureAlgorithms` are appended to `sshd_config`, using the same `lineinfile` plus `sshd -t` plus backup pattern used everywhere else. +This matters because Ubuntu cloud images ship exactly that `50-cloud-init.conf`. Editing only the +main `sshd_config`, as this role does by default, leaves password login enabled on those hosts. -If `ssh_key_rotation_manage_crypto_policy` is set and the target has the tooling, the crypto-policy step runs there too. Because `update-crypto-policies --set` validates only its own module syntax and not the resulting merged `sshd_config`, there is an explicit `sshd -t` re-check afterwards, before the reload handler is allowed to fire. The drop-in warning also calls out `50-redhat.conf` by name, since that file is the generated crypto-policy backend include and is not meant to be hand-edited. +**This role writes drop-ins too, and never edits your `sshd_config`.** On any host with an +`Include` line it writes two files: -Once `sshd` has reloaded, the `sshd -T` check flags any requested algorithm that still is not showing up in the effective config, so you find out before Phase 2 tries and fails to reconnect. +``` +/etc/ssh/sshd_config.d/99-ssh-key-rotation.conf # install stage +/etc/ssh/sshd_config.d/99-ssh-key-rotation-lockdown.conf # verify stage +``` -### Combining a base policy with a subpolicy module +Two files rather than one so that a verify-stage rollback cannot delete install-stage settings a +previous run legitimately established. Hosts older than OpenSSH 8.2 have no `Include`, so there the +role falls back to a marked block inside `sshd_config`, backed up first. -`update-crypto-policies --set` accepts either a base policy name on its own (`DEFAULT`, `FIPS`, `LEGACY` and so on) or a base policy combined with one or more subpolicy *modules*, written as `BASE:MODULE`. For example `FIPS:PQ`, or `FIPS:PQ:NO-SHA1` to stack more than one. +The `99-` default is chosen so **your** drop-ins still outrank this role's. That keeps your intent +authoritative, and it is what lets Phase 2 detect a conflict and fail loudly rather than quietly +working around you. If a drop-in of yours defeats the lock-down you have two options: -Each module is a `MODULE.pmod` file, either shipped by the OS under `/usr/share/crypto-policies/policies/modules/` or dropped in locally under `/etc/crypto-policies/policies/modules/`. A module is added on top of the base policy rather than replacing it, so `FIPS:PQ` stays FIPS-compliant everywhere else and only adds what `PQ.pmod` grants. +**Fix the drop-in yourself** and re-run. Best when it is something you manage. -This matters specifically for PQC. On AlmaLinux and RHEL 9, the `FIPS` policy alone includes no post-quantum key-exchange groups, but the OS still ships a built-in `PQ.pmod` that adds `mlkem768x25519-sha256` and other ML-KEM groups when combined as `FIPS:PQ`. This was confirmed against a real AlmaLinux 9.8 host, where `sshd -T` only showed `mlkem768x25519-sha256` in the effective `KexAlgorithms` after switching from `FIPS` to `FIPS:PQ`. AlmaLinux and RHEL 10 ship PQC key exchange in `FIPS` already, so the combination is not needed there. +**Let the role outrank it,** by lowering the prefix: -Before ever calling `update-crypto-policies --set`, the playbook lists whatever `*.pmod` files exist under both module directories on that host, the control node in Phase 0 and the target in Phase 1. If `ssh_key_rotation_crypto_policy_setting` names a module that is not present, it fails before making any change, rather than letting `update-crypto-policies` silently ignore an unknown module name or fail in a way that is easy to miss in the task output. +```yaml +ssh_key_rotation_sshd_dropin_prefix: "01" +``` -### Example: enabling a PQC key-exchange algorithm +Understand the trade before you do. At `01-` this role's files win, which also means the +"lock-down did not take effect" check can no longer detect a conflicting drop-in - the role's file +always wins, so the check becomes a formality. It is also why the role never manages +`AuthorizedKeysFile`: at that precedence it would override a central key store such as +`AuthorizedKeysFile /etc/ssh/authorized_keys/%u` and strip key access from every *other* user on +the host, while the rotated user kept working. The role reads that setting and fails instead. -```bash -ansible-playbook krameff.ssh_key_rotation.rotate \ - -i inventory.ini \ - -e old_private_key=~/.ssh/id_old \ - -e new_private_key=~/.ssh/pwc_id_ed25519 \ - -e new_public_key_file=./pwc_id_ed25519.pub \ - -e old_public_key_file=./id_old.pub \ - -e '{"ssh_key_rotation_pqc_kex_algorithms": ["mlkem768x25519-sha256"]}' -``` +### Match blocks -### Example: RHEL/Fedora crypto-policy management +A `Match` block is a separate problem with the same shape. `sshd -T` on its own reports the global +configuration and does not evaluate `Match` at all, so a block like: -```bash -ansible-playbook krameff.ssh_key_rotation.rotate \ - -i inventory.ini \ - -e old_private_key=~/.ssh/id_old \ - -e new_private_key=~/.ssh/pwc_id_ed25519 \ - -e new_public_key_file=./pwc_id_ed25519.pub \ - -e old_public_key_file=./id_old.pub \ - -e ssh_key_rotation_manage_crypto_policy=true \ - -e ssh_key_rotation_crypto_policy_setting=DEFAULT:PQ +``` +Match User deploy + PasswordAuthentication yes ``` -### Example: adding PQC to a FIPS-mode host +leaves password login working for exactly the account you just rotated, while the global check +says it is disabled. Phase 2 therefore also runs `sshd -T -C user=,...`, which does +evaluate `Match`, and fails if password or keyboard-interactive auth is still enabled for that +user. Turn it off with `ssh_key_rotation_check_match_blocks: false` if `sshd -T -C` misbehaves on +your hosts. -```bash -ansible-playbook krameff.ssh_key_rotation.rotate \ - -i inventory.ini \ - -e old_private_key=~/.ssh/id_old \ - -e new_private_key=~/.ssh/pwc_id_ecdsa \ - -e new_public_key_file=./pwc_id_ecdsa.pub \ - -e old_public_key_file=./id_old.pub \ - -e ssh_key_rotation_manage_crypto_policy=true \ - -e ssh_key_rotation_crypto_policy_setting=FIPS:PQ +**The role never edits a `Match` block.** Everything it writes to `sshd_config` goes into two +clearly marked blocks in the global section: + +``` +# BEGIN krameff.ssh_key_rotation install +# BEGIN krameff.ssh_key_rotation lock-down ``` +Both are anchored immediately after the `Include` line, or before the first `Match` on hosts with +no `Include`. If a `Match` block contradicts what the role set, the role reports it and stops +rather than editing someone's per-user policy. See [Limitations](#limitations). + ## Safety model The point of this playbook is that you should not be able to lock yourself out by running it. That comes down to a handful of rules it never breaks. @@ -338,12 +341,75 @@ The point of this playbook is that you should not be able to lock yourself out b Resetting the connection first matters if your `ansible.cfg` enables SSH `ControlPersist`. That multiplexes connections by host, port and user, not by identity file, so without the reset, Phase 2 could silently ride on Phase 1's still-open connection instead of genuinely testing the new key. +**The lock-down is checked after the fact, not assumed.** `sshd_config` has `Include /etc/ssh/sshd_config.d/*.conf` at the top on every mainstream distribution, and sshd honours the *first* value it sees for a keyword. A drop-in therefore beats anything this playbook writes further down the file. Phase 2 re-reads `sshd -T` after its edits and fails, naming the problem, rather than reporting a lock-down that did not happen. This is a global check: `sshd -T` without `-C` does not evaluate `Match` blocks, which is why those get their own warning in Phase 1. + +**`authorized_keys` ownership is checked, on both the success and the rollback path.** sshd's `StrictModes` silently ignores an `authorized_keys` file that is not owned by the target user, which rejects every key at once. Because the playbook does its file work under `become`, ownership is set explicitly and then asserted. + +**Connectivity checks reconnect from scratch.** Every check that claims a key still works first resets the connection, so it cannot pass by riding an SSH `ControlPersist` session opened earlier in the run. + **Every `sshd_config` change is validated with `sshd -t` before it is written.** **Configuration is applied with a reload, never a restart,** so sessions already open stay open. **`sshd_config` and `authorized_keys` are backed up before every edit,** so you can always roll back by hand. +### What a failed run undoes + +```mermaid +flowchart TD + Start["Rotation starts"] --> Record["Install records what it will change + in /etc/ansible/facts.d/ssh_key_rotation.fact"] + Record --> Install["Install: add the new key, + write this role's drop-in"] + + Install --> InstallOK{"Install succeeded?"} + InstallOK -->|"No"| IRB["Install rollback: + remove this role's drop-in, + restore authorized_keys, + restore the crypto-policy"] + IRB --> IProbe["Prove the old key still logs in + (real ssh, no multiplexing)"] + IProbe --> Failed["Fail loudly, host as we found it"] + + InstallOK -->|"Yes"| Gate{"Does the NEW key authenticate?"} + Gate -->|"No"| Abort["Abort - old key never touched"] + Gate -->|"Yes"| Verify["Verify: remove the old key, + write the lock-down drop-in"] + + Verify --> VerifyOK{"Lock-down verified? + sshd -T, per-user Match check, + authorized_keys ownership"} + VerifyOK -->|"Yes"| Done["Done - rotation complete"] + VerifyOK -->|"No"| VRB["Verify rollback: + remove both drop-ins, + restore authorized_keys + (old key back, new key kept)"] + VRB --> VProbe["Prove access on a fresh, + unmultiplexed connection"] + VProbe --> Failed +``` + +Note what is *not* in the diagram: editing `/etc/ssh/sshd_config`. On any host with an `Include` +line the role only ever adds and removes its own files, which is what makes "undo" a deletion +rather than a restore. + +**A failed run puts the host back.** Both the install and the verify stage roll back their own +changes: this role's drop-ins are removed (or restored, if a file was already at that path), and +`authorized_keys` is restored from the backup taken before the run. What each stage actually did is +recorded on the host at `/etc/ansible/facts.d/ssh_key_rotation.fact`, so the rollback undoes exactly +that rather than guessing - and so it still works when the verify stage is run on its own. + +Two deliberate exceptions. The new key is **left** in `authorized_keys` by default: it is a +credential you hold, and removing it would mean relying on the old key still working, which is not +guaranteed. Set `ssh_key_rotation_rollback_remove_new_key: true` to restore the file exactly, in +which case the old key is proven to work *before* anything is removed, and kept if that proof fails. +The state file and the timestamped backups are also left behind, as the record of what happened. + +**Rollback proves access with a real SSH connection.** Not `ansible.builtin.ping`: Ansible +multiplexes connections on host, port and user rather than on the identity file, so a ping can +succeed over a socket opened with a different credential and report access that no longer exists. +The check runs a real `ssh` with `ControlMaster=no` and `ControlPath=none`. + **Phase 2's cleanup runs inside a `block`/`rescue`.** If removing the old key or disabling legacy auth fails partway through, `rescue` restores `authorized_keys` and `sshd_config` from the backups just taken, reloads sshd, re-confirms connectivity, and fails with a clear message. All of that happens on the same still-open connection, before it could be lost. A reload that itself fails does not abort the rollback. The restored files are already on disk and sshd reads `authorized_keys` per connection, so the rollback finishes and the failure message tells you to reload sshd by hand. @@ -380,7 +446,9 @@ A reload that itself fails does not abort the rollback. The restored files are a 7. Optionally make `authorized_keys` exclusive to the new key. 8. Disable password and keyboard-interactive auth if requested, backing up `sshd_config` first. 9. Apply the final sshd configuration. -10. Run one last connectivity check. +10. Re-read `sshd -T` and confirm password and keyboard-interactive auth really are disabled, failing if a drop-in overrode them. +11. Confirm `authorized_keys` is still owned by the target user and not group- or world-writable, so sshd's `StrictModes` will honour it. +12. Reset the connection and run one last connectivity check, so it re-authenticates rather than reusing the open session. ## Role reference @@ -418,6 +486,50 @@ There is deliberately no `tasks/main.yml` entry point, because each stage authen - ansible.builtin.include_role: {name: ssh_key_rotation, tasks_from: verify} ``` +## Limitations + +Worth knowing before you rely on this in production. None of these cause silent failures: where +the role cannot do something, it stops and says so rather than reporting a success it did not +achieve. + +**`Match` blocks are never edited.** Everything written to `sshd_config` goes into two marked +blocks in the global section. This is deliberate. A `Match` block is a per-user or per-address +policy someone set on purpose, and `PubkeyAuthentication`, `AuthorizedKeysFile`, +`PasswordAuthentication` and `PubkeyAcceptedAlgorithms` are all legal inside one. If a `Match` +block contradicts the lock-down, Phase 2 fails and names it; fix the block by hand and re-run. + +**The `Match` check evaluates one connection profile, not all of them.** Phase 2 runs +`sshd -T -C user=,host=localhost,addr=`. That catches `Match User` and +`Match Address` blocks affecting the rotation's own connection. A block keyed on something else, +such as `Match LocalPort` or an address range the rotation did not come from, is not evaluated +and could still apply to a future login. + +**Your drop-in files are never modified.** The role adds its own files under +`/etc/ssh/sshd_config.d/` and removes them again on rollback, but it will not edit a drop-in you +already have. If one at its own path already exists it is backed up and restored rather than +deleted. See [Drop-in overrides](#drop-in-overrides). + +**Crypto-policy changes are not rolled back byte-for-byte.** `update-crypto-policies --set` +regenerates `/etc/crypto-policies/back-ends/*`, so restoring the previous policy *name* does not +restore files an administrator hand-edited underneath it. The control node's own policy, if +`ssh_key_rotation_manage_crypto_policy` changed it, is not restored at all. + +**The state file is left on the host** at `/etc/ansible/facts.d/ssh_key_rotation.fact`, deliberately. +It records what the last run changed, which is what makes a later recovery possible once the +original connection is gone. + +**Only one account is rotated per run,** the one in `ssh_key_rotation_target_user`. Other users' +`authorized_keys` files are untouched. + +**Recovery depends on the connection staying up.** Phase 2's rollback restores `authorized_keys` +and `sshd_config` over the connection it already holds. If that connection is lost at the wrong +moment, there is no remote path back in; you need console access. This is why the collection +insists on proving the new key before removing the old one, and why a snapshot is worth taking. + +**`sshd -T` is trusted for verification.** Where a host's `sshd -T` cannot run at all, the +lock-down checks are skipped with a warning rather than failing the run, since the alternative is +rolling back a rotation that actually succeeded. + ## Troubleshooting ### Phase 0 validation failures @@ -457,11 +569,50 @@ Phase 2 could not connect with the new key. Things to check: ### "Drop-in sshd config files found" -Phase 1 spotted override files in `/etc/ssh/sshd_config.d/`. Worth reviewing: +A warning, not a failure. Phase 1 spotted override files in `/etc/ssh/sshd_config.d/`. Worth reviewing: 1. See what is there: `ansible all -i inventory.ini -m ansible.builtin.find -a "paths=/etc/ssh/sshd_config.d patterns='*.conf'" -b` 2. Make sure none of them re-enable `PasswordAuthentication yes` or similar. -3. If needed, update the drop-ins by hand before running Phase 2, or pass `ssh_key_rotation_make_exclusive=false` to keep the old key active a little longer. +3. If one does, fix it there before re-running. Phase 2 will fail rather than let it slide, as below. + +### "sshd -T still reports password authentication as ENABLED" + +Phase 2 disabled password auth in `/etc/ssh/sshd_config`, then re-read the effective config and +found it still on, so it rolled back and failed. The lock-down did not take effect. + +Almost always a drop-in. `sshd_config` has `Include /etc/ssh/sshd_config.d/*.conf` at the top, +and sshd uses the **first** value it sees for a keyword, so a drop-in beats anything written +further down the main file. Ubuntu cloud images ship exactly this, as +`50-cloud-init.conf` containing `PasswordAuthentication yes`. + +To fix, set the value in the winning drop-in, or remove it there, and re-run: + +```bash +sudo grep -rn PasswordAuthentication /etc/ssh/sshd_config.d/ +sudo sshd -T | grep -i passwordauthentication # what is actually in effect +``` + +The rotation itself succeeded before this check; the rollback put the old key and the previous +`sshd_config` back, so the host is exactly as it started. If you would rather not disable +password auth at all on these hosts, set `ssh_key_rotation_disable_password_auth: false` and the +check is skipped with it. + +### "StrictModes will ignore this file, rejecting every key for this user" + +`~/.ssh/authorized_keys` is not owned by the user being rotated, or is group- or world-writable. +sshd refuses to read such a file at all, so *every* key for that account stops working, not just +the new one. + +The playbook sets ownership explicitly and asserts it, so seeing this means something else on the +host changed it. Fix it directly: + +```bash +sudo chown : ~/.ssh/authorized_keys +sudo chmod 600 ~/.ssh/authorized_keys +``` + +Check the backups beside it too (`authorized_keys.bak-*`), since those are what a future rollback +would restore from. ### PQC algorithms not negotiating @@ -480,7 +631,7 @@ This is a hard stop before `update-crypto-policies --set` is ever called, on the 1. List what is actually available: `ssh ls /usr/share/crypto-policies/policies/modules/*.pmod /etc/crypto-policies/policies/modules/*.pmod` 2. Check for typos in the module name, or drop a custom `MODULE.pmod` into `/etc/crypto-policies/policies/modules/` if you need one the OS does not ship. -3. See [Combining a base policy with a subpolicy module](#combining-a-base-policy-with-a-subpolicy-module) for why `FIPS:PQ` is the combination most people want on RHEL and AlmaLinux 9. +3. See [Combining a base policy with a subpolicy module](PQC.md#combining-a-base-policy-with-a-subpolicy-module) for why `FIPS:PQ` is the combination most people want on RHEL and AlmaLinux 9. ### "Permission denied" on Phase 1 @@ -508,13 +659,16 @@ Pipelining is the more robust fix, since some hardening baselines make every wri ansible-galaxy collection build . # Test against a local VM, brought up however you like -ansible-playbook playbooks/rotate.yml -i 127.0.0.1, \ - -e old_private_key=~/.ssh/id_rsa \ - -e new_private_key=~/.ssh/id_ed25519 \ - -e new_public_key_file=./id_ed25519.pub \ - -e old_public_key_file=./id_rsa.pub +ansible-playbook playbooks/rotate.yml -i 127.0.0.1, -e @rotation_vars.yml + +# Run the container test suites +molecule test -s default # full rotation, twice, proving idempotency +molecule test -s nonroot # same, rotating a non-root user +molecule test -s rollback # breaks a rotation mid-verify to prove access is restored ``` +See [DEVELOPMENT.md](DEVELOPMENT.md) for the repository layout and release process. + ## Contributing Issues and bug reports are welcome and genuinely useful. diff --git a/rotation_vars.example.yml b/rotation_vars.example.yml new file mode 100644 index 0000000..5ea8981 --- /dev/null +++ b/rotation_vars.example.yml @@ -0,0 +1,33 @@ +--- + +# Example extra-vars file for a rotation. Copy it, fill in your own paths, and pass it with a +# single -e instead of four: +# +# cp rotation_vars.example.yml rotation_vars.yml +# ansible-playbook -i inventory.ini playbooks/rotate.yml -e @rotation_vars.yml +# +# Your copy is gitignored (see .gitignore). These are paths on the CONTROL NODE, not the target +# hosts, and all four are required - the validate stage refuses to start without them. +# +# They are intentionally NOT set in roles/ssh_key_rotation/defaults/main.yml: anything defined +# there counts as "defined", which would permanently disable the check that catches a missing or +# misspelled variable before a single host is touched. + +# The key you can currently log in with. Used by the install stage to reach each host, and +# removed from authorized_keys by the verify stage once the new key is proven to work. +old_private_key: "~/.ssh/id_old" +old_public_key_file: "~/.ssh/id_old.pub" + +# The key you are rotating TO. The verify stage reconnects with this one before anything is +# deleted, so a bad path here fails safely rather than locking you out. +new_private_key: "./pwc_id_ed25519" +new_public_key_file: "./pwc_id_ed25519.pub" + +# Optional. The account whose authorized_keys is rotated; defaults to the inventory's +# ansible_user. Both keys above must belong to this same user. +# ssh_key_rotation_target_user: "ubuntu" + +# Optional cleanup settings, shown with their defaults. +# ssh_key_rotation_disable_password_auth: true +# ssh_key_rotation_disable_kbd_interactive: true +# ssh_key_rotation_make_exclusive: false # true leaves ONLY the new key in authorized_keys From d5bf7c7bf19e9a4d90b0e49f42c06d11d41a7e7b Mon Sep 17 00:00:00 2001 From: Mark Bolwell Date: Thu, 6 Aug 2026 19:54:00 +0100 Subject: [PATCH 16/18] Version and changelog Signed-off-by: Mark Bolwell --- .github/ISSUE_TEMPLATE/bug_report.yml | 2 +- CHANGELOG.md | 71 +++++++++++++++++++++------ galaxy.yml | 6 ++- 3 files changed, 60 insertions(+), 19 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index d80c8e3..425cdf0 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -93,7 +93,7 @@ body: id: collection-version attributes: label: Collection version - placeholder: "1.0.0, or a commit SHA if installed from source" + placeholder: "0.9.0, or a commit SHA if installed from source" validations: required: true diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d7c03e..20f01d7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,35 +6,74 @@ - Release workflow: tagging `v*` builds the collection and publishes it to Galaxy, needs a `GALAXY_API_KEY` secret. - Release workflow fails the build if the tag doesn't match the version in `galaxy.yml`. +- Releases attach the collection tarball to a GitHub Release and attest its build provenance, so it can be traced back to this repository. - `CONTRIBUTING.md`, covering setup, the required checks and the pull request checklist. - `SECURITY.md`, with a private disclosure route for anything that could leave a host unreachable. - GitHub issue forms and a pull request template under `.github/`. - Pre-commit configuration for secret detection, whitespace, YAML and ansible-lint. -- Releases now attach the collection tarball to a GitHub Release and attest its build provenance, so it can be traced back to this repository. +- `rotation_vars.example.yml`: copy it, fill in your key paths, and pass `-e @rotation_vars.yml` instead of four separate `-e` flags. -### Fixed +#### Safety checks -- Verify stage rollback no longer aborts if the sshd reload fails - it finishes restoring, re-confirms connectivity, and reports that sshd needs a manual reload. -- Boolean role variables are coerced with `| bool`, so passing them via `-e` (e.g. `-e ssh_key_rotation_disable_password_auth=false`) no longer fails on ansible-core 2.20. -- Molecule old-key-removal check matched base64 key material as a regex, so it could pass while the old key was still authorized. -- Molecule key-authentication checks ran over the container connection, which ignores SSH keys and so could never detect a failure; they now drive a real ssh client. -- Molecule idempotency check required `changed=0` from the timestamped backup tasks, which cannot be idempotent by design; it now allows those and fails on anything else. -- Molecule rollback check compared `authorized_keys` against a checkpoint taken before the install stage, demanding the rescue undo work it never performs. -- Molecule rollback "sshd is running" check hardcoded the `sshd` unit, which resolves on Ubuntu only through an alias; it now uses the same family lookup as prepare. -- Molecule set `ANSIBLE_ROLES_PATH` to a scenario-relative path (`../../../roles`), but Molecule runs with the project directory as its cwd, so it resolved outside the repo entirely and the rollback scenario could not find the role. The collection's own `ansible.cfg` cannot cover for it either, since Molecule generates its own `ansible.cfg` and points `ANSIBLE_CONFIG` at it. It is now `${MOLECULE_PROJECT_DIRECTORY:-.}/roles`, which is the collection root in both cases. -- The default scenario's first verify play was named as a new-key authentication check but runs over the container connection, which ignores SSH keys, so it could never fail; it is now named and documented as the reachability check it actually is. The key-authentication proof was already in the following play, which drives a real ssh client. +- Verify stage confirms with `sshd -T` that password and keyboard-interactive logins really were turned off, rather than trusting the config edit. +- Verify stage confirms the lock-down applies to the rotated user specifically, via `sshd -T -C`, catching a `Match` block that re-enables password login for exactly that account. +- Verify stage checks `authorized_keys` ownership and permissions still satisfy sshd's `StrictModes`, on both the success and rollback paths. +- Install stage reads the effective `AuthorizedKeysFile` and fails if the host does not read keys from the user's own `~/.ssh/authorized_keys`, rather than installing a key somewhere sshd will never look. + +#### Rollback + +- The install stage now rolls back too. A failure there - the crypto-policy guard refusing a key type, for instance - undoes the new key and this role's configuration instead of leaving them behind. +- A rollback restores the old key. Both stages put `authorized_keys` back as it was before the run, and by default leave the new key alongside it, so the host never ends up depending solely on the old key still working. +- `ssh_key_rotation_rollback_remove_new_key` (default `false`) restores `authorized_keys` exactly, removing the new key. The old key is proven to work first, and the new key is kept if that proof fails. +- The role records what it changed on each host in `/etc/ansible/facts.d/ssh_key_rotation.fact`, so a rollback undoes exactly that, and so the verify stage can roll back install-stage changes even when run on its own. Left in place after a run as a record of what happened. + +#### Variables + +- `ssh_key_rotation_sshd_dropin_prefix` (default `"99"`) and `ssh_key_rotation_sshd_dropin_dir`: control where the role writes its configuration. Lower prefixes win, because sshd keeps the first value it sees for a keyword. +- `ssh_key_rotation_manage_sshd_dropin` is deprecated in favour of the prefix variable, but still honoured; it is equivalent to a prefix of `01`. +- `ssh_key_rotation_check_match_blocks` (default `true`) enables the per-user `Match` block check. + +#### Testing + +- New `nonroot` Molecule scenario rotates a non-root user, and `rollback` now does too. The previous root-only scenarios could never catch a file-ownership mistake, because when the target is root, root-owned is the correct outcome. +- Molecule fails if a container goes missing from the generated inventory, instead of quietly testing fewer hosts than it claims. +- Reusable test-harness agent definitions under `.claude/agents/`, carrying the rules that stop a test lying: never prove connectivity over a multiplexed connection, and never report "refused" when the honest answer is "could not connect". ### Changed +- On any host with an `Include` line (OpenSSH 8.2+, so every currently supported distribution) the role writes its settings to drop-ins under `/etc/ssh/sshd_config.d/` and no longer edits `/etc/ssh/sshd_config` at all, so rolling back is deleting a file rather than restoring someone else's config. Two files are used - one per stage - so a verify-stage rollback cannot delete install-stage settings an earlier run established. Older hosts keep the marked-block behaviour, backed up first. This does add files to `/etc/ssh/sshd_config.d/`, which config-drift tooling such as AIDE or Tripwire will notice. Nothing has been released yet, so no published behaviour changes. +- The role no longer sets `AuthorizedKeysFile`. Forcing it would have overridden a central key store at drop-in precedence, stripping key access from every other user on the host while the rotated user kept working. +- QUICKSTART and README rewritten around the vars file, with the repeated four-flag command replaced by a single example, plus a Limitations section and the new failure messages. +- Post-quantum documentation moved out of README into [PQC.md](PQC.md), leaving a short pointer behind. - Molecule scenarios moved from `molecule/` to `extensions/molecule/`, which also corrects the relative roles path. - Molecule test images install `openssh-server` at build time rather than during prepare. -- Both Molecule scenarios now share one `extensions/molecule/resources/Dockerfile.j2` and one `resources/prepare.yml`; each scenario's `prepare.yml` is a short wrapper that supplies only its own keypairs and which one to pre-authorize. -- Root's Ansible `remote_tmp` directory is created during the image build instead of by an entrypoint wrapper, so it exists before the container ever starts and `molecule-entrypoint.sh` is gone from both scenarios. +- Molecule scenarios share one `extensions/molecule/resources/Dockerfile.j2` and one `resources/prepare.yml`; each scenario's `prepare.yml` is a short wrapper supplying only its own keypairs and which one to pre-authorize. +- Root's Ansible `remote_tmp` directory is created during the image build instead of by an entrypoint wrapper, so it exists before the container starts and `molecule-entrypoint.sh` is gone. - Molecule platform definitions use a YAML anchor rather than repeating three near-identical blocks per scenario. -- Molecule test sequences start with `dependency` and `destroy`: collection dependencies are installed the same way locally and in CI (the workflow's separate `ansible-galaxy` step is gone), and a container left behind by an aborted run can no longer be reused with keys already in its `authorized_keys`. +- Molecule test sequences start with `dependency` and `destroy`, so collection dependencies install the same way locally and in CI, and a container left behind by an aborted run cannot be reused with keys already in its `authorized_keys`. - The Molecule workflow also runs weekly, so drift in the `:latest` test images surfaces on its own build rather than on an unrelated pull request. -- `build_ignore` still pointed at `molecule/`, so the scenarios were being shipped in the artifact; it now excludes `extensions/molecule` plus `.github`, `.claude`, `.cursor`, `.ansible`, `.mcp.json` and `inventory.ini`. -- `DEVELOPMENT.md` shows how to generate throwaway keys outside the repo for manual testing. +- `build_ignore` still pointed at `molecule/`, so the scenarios were being shipped in the artifact; it now excludes `extensions/molecule` plus `.github`, `.claude`, `.cursor`, `.ansible`, `.mcp.json`, `inventory.ini` and `rotation_vars.yml`. +- `DEVELOPMENT.md` shows how to generate throwaway keys outside the repo for manual testing, and states that behaviour may change within `0.x` until 1.0.0 ships. + +### Fixed + +- A failed install stage left the host carrying the new key and this role's sshd configuration while reporting that "nothing has been broken". Access was intact, but the host was not as it was found. Both are now undone. +- Rollback left the restored `authorized_keys` owned by root, which sshd ignores under `StrictModes` - so the rollback meant to save your access could lock you out instead. Ownership is now restored explicitly, using the target user's real primary group. +- Rollback connectivity checks used `ansible.builtin.ping`, whose connection is multiplexed on host, port and user rather than on the identity file, so they could pass over a socket opened with a different credential and report access that no longer existed. They now drive a real ssh client with `ControlMaster=no` and `ControlPath=none`, and force public key authentication so a client-side `ssh_config` cannot make a good key look broken. +- A failed connectivity check during rollback hid the message naming the restored backup files, exactly when you most need it. That message is now always shown. +- Every `sshd_config` setting was written with `lineinfile`, which replaces the LAST match, and `Match` blocks sit at the end of the file - so on a host with such a block both stages edited per-user policy instead of the global section, silently flipping settings like a deliberate `PubkeyAuthentication no` for one account. The role now writes drop-ins, or a block anchored to the global section on older hosts, and never edits a `Match` block. Documented under Limitations in the README. +- A drop-in under `/etc/ssh/sshd_config.d/` (cloud-init's, typically) could silently override the lock-down and leave password logins working while the run reported success. That is now caught and named. +- Verify stage rollback no longer aborts if the sshd reload fails - it finishes restoring, re-confirms access, and reports that sshd needs a manual reload. Reloads are now skipped entirely when no configuration file changed, since `authorized_keys` is read per connection and needs none. +- Timestamped backup names use `ansible_facts.date_time`, clearing a deprecation warning ahead of ansible-core 2.24. +- Boolean role variables are coerced with `| bool`, so passing them via `-e` (e.g. `-e ssh_key_rotation_disable_password_auth=false`) no longer fails on ansible-core 2.20. +- Molecule built its test inventory with concurrent appends, which could silently drop a host; it is now written in one pass and asserted complete. +- Molecule old-key-removal check matched base64 key material as a regex, so it could pass while the old key was still authorized. +- Molecule key-authentication checks ran over the container connection, which ignores SSH keys and so could never detect a failure; they now drive a real ssh client. +- Molecule idempotency check required `changed=0` from tasks that record a per-run timestamp and so can never be idempotent; it now allows those and fails on anything else. +- Molecule rollback check compared `authorized_keys` against a checkpoint taken before the install stage, demanding the rescue undo work it never performs. +- Molecule rollback "sshd is running" check hardcoded the `sshd` unit, which resolves on Ubuntu only through an alias; it now uses the same family lookup as prepare. +- Molecule set `ANSIBLE_ROLES_PATH` to a scenario-relative path, but Molecule runs with the project directory as its cwd, so it resolved outside the repo entirely and the rollback scenario could not find the role. It is now `${MOLECULE_PROJECT_DIRECTORY:-.}/roles`. +- The default scenario's first verify play was named as a new-key authentication check but runs over the container connection, which ignores SSH keys, so it could never fail; it is now named and documented as the reachability check it actually is. ### Removed diff --git a/galaxy.yml b/galaxy.yml index 69d3c2b..988809c 100644 --- a/galaxy.yml +++ b/galaxy.yml @@ -2,7 +2,7 @@ namespace: krameff name: ssh_key_rotation -version: 1.0.0 +version: 0.9.0 readme: README.md authors: - Krameff Solutions Limited @@ -28,5 +28,7 @@ build_ignore: - .mcp.json - inventory.ini # Belt and braces: these are gitignored and should never exist in a clean checkout, but a - # build run from a developer's working tree must not pick up local key material. + # build run from a developer's working tree must not pick up local key material or paths. + # rotation_vars.example.yml is NOT excluded - the docs tell users to copy it. - test_* + - rotation_vars.yml From c050eda0c529f37fb8e0ae6909d92f604251bb2b Mon Sep 17 00:00:00 2001 From: Mark Bolwell Date: Thu, 6 Aug 2026 19:54:53 +0100 Subject: [PATCH 17/18] updated Signed-off-by: Mark Bolwell --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 6d9b161..9b8356e 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,7 @@ venv/ .idea/ .vscode/ .cursor/ +.claude/ *.iml .vagrant/ *.log From c18a0c63b22c60c6d0f52083d5906d25ec11465b Mon Sep 17 00:00:00 2001 From: Mark Bolwell Date: Thu, 6 Aug 2026 19:55:29 +0100 Subject: [PATCH 18/18] logic update Signed-off-by: Mark Bolwell --- extensions/molecule/default/converge.yml | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/extensions/molecule/default/converge.yml b/extensions/molecule/default/converge.yml index f2661da..602e960 100644 --- a/extensions/molecule/default/converge.yml +++ b/extensions/molecule/default/converge.yml @@ -43,11 +43,17 @@ changed_when: true register: ssh_key_rotation_second_run - # The three "Back up ..." tasks write to a dest containing ansible_date_time, so they produce a - # new file on every run and can never report changed=0. That's deliberate: each run must - # snapshot the state before that run, and reusing a fixed filename would overwrite the original - # pre-rotation backup. So they are expected to change, and anything else changing is a real - # idempotency regression - which is what this splits apart. + # Two kinds of task are expected to change on every run and are excluded here; anything else + # changing is a real idempotency regression, which is what this splits apart. + # + # "Back up ..." writes to a dest containing a timestamp, so it produces a new file each run. + # That is deliberate: each run must snapshot the state before that run, and a fixed filename + # would overwrite the original pre-rotation backup. + # + # "Write the rotation state fact" records run_id, a fresh timestamp per run, so its content + # differs every time by design. That file is what lets a rollback undo exactly what this run + # did, and what makes recovery possible after the connection is gone, so it cannot be made + # content-stable without losing the thing it is for. - name: Converge | Split the second run's changed tasks into expected backups vs unexpected vars: ssh_key_rotation_second_run_json: "{{ ssh_key_rotation_second_run.stdout | from_json }}" @@ -57,7 +63,8 @@ {%- for play in ssh_key_rotation_second_run_json.plays -%} {%- for task in play.tasks -%} {%- for host, result in task.hosts.items() -%} - {%- if (result.changed | default(false)) and 'Back up' not in task.task.name -%} + {%- if (result.changed | default(false)) and 'Back up' not in task.task.name + and 'rotation state fact' not in task.task.name -%} {%- set _ = found.append(task.task.name ~ ' on ' ~ host) -%} {%- endif -%} {%- endfor -%} @@ -69,7 +76,8 @@ {%- for play in ssh_key_rotation_second_run_json.plays -%} {%- for task in play.tasks -%} {%- for host, result in task.hosts.items() -%} - {%- if (result.changed | default(false)) and 'Back up' in task.task.name -%} + {%- if (result.changed | default(false)) and ('Back up' in task.task.name + or 'rotation state fact' in task.task.name) -%} {%- set _ = found.append(task.task.name) -%} {%- endif -%} {%- endfor -%}