diff --git a/.eslintrc.json b/.eslintrc.json index 04820b63e..7956d0898 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -65,7 +65,11 @@ }, { "sourceTag": "scope:backend", - "onlyDependOnLibsWithTags": ["scope:backend", "domain:shared"] + "onlyDependOnLibsWithTags": [ + "scope:backend", + "scope:shared", + "domain:shared" + ] }, { "sourceTag": "scope:keycloak", @@ -96,6 +100,13 @@ "domain:identity", "domain:shared" ] + }, + { + "sourceTag": "domain:loadweaver", + "onlyDependOnLibsWithTags": [ + "domain:loadweaver", + "domain:shared" + ] } ] } diff --git a/.gitignore b/.gitignore index fcac1ce64..b1ec271ec 100644 --- a/.gitignore +++ b/.gitignore @@ -84,3 +84,4 @@ opencode.json # Environment variables *.env +loadweaver.yml diff --git a/apps/loadweaver/cli-loadweaver/.eslintrc.json b/apps/loadweaver/cli-loadweaver/.eslintrc.json new file mode 100644 index 000000000..3456be9b9 --- /dev/null +++ b/apps/loadweaver/cli-loadweaver/.eslintrc.json @@ -0,0 +1,18 @@ +{ + "extends": ["../../../.eslintrc.json"], + "ignorePatterns": ["!**/*"], + "overrides": [ + { + "files": ["*.ts", "*.tsx", "*.js", "*.jsx"], + "rules": {} + }, + { + "files": ["*.ts", "*.tsx"], + "rules": {} + }, + { + "files": ["*.js", "*.jsx"], + "rules": {} + } + ] +} diff --git a/apps/loadweaver/cli-loadweaver/jest.config.cts b/apps/loadweaver/cli-loadweaver/jest.config.cts new file mode 100644 index 000000000..5bdbf91f7 --- /dev/null +++ b/apps/loadweaver/cli-loadweaver/jest.config.cts @@ -0,0 +1,10 @@ +module.exports = { + displayName: 'loadweaver-cli-loadweaver', + preset: '../../../jest.preset.cjs', + testEnvironment: 'node', + transform: { + '^.+\\.[tj]s$': ['ts-jest', { tsconfig: '/tsconfig.spec.json' }], + }, + moduleFileExtensions: ['ts', 'js', 'html'], + coverageDirectory: '../../../coverage/apps/loadweaver/cli-loadweaver', +}; diff --git a/apps/loadweaver/cli-loadweaver/project.json b/apps/loadweaver/cli-loadweaver/project.json new file mode 100644 index 000000000..cda5e7a42 --- /dev/null +++ b/apps/loadweaver/cli-loadweaver/project.json @@ -0,0 +1,145 @@ +{ + "name": "loadweaver-cli-loadweaver", + "$schema": "../../../node_modules/nx/schemas/project-schema.json", + "sourceRoot": "apps/loadweaver/cli-loadweaver/src", + "projectType": "application", + "tags": ["type:app", "scope:shared", "domain:loadweaver"], + "targets": { + "build": { + "executor": "@nx/esbuild:esbuild", + "outputs": ["{options.outputPath}"], + "defaultConfiguration": "production", + "options": { + "platform": "node", + "outputPath": "dist/apps/loadweaver/cli-loadweaver", + "format": ["cjs"], + "bundle": true, + "main": "apps/loadweaver/cli-loadweaver/src/main.ts", + "tsConfig": "apps/loadweaver/cli-loadweaver/tsconfig.app.json", + "assets": [ + { + "input": "libs/domains/loadweaver/shared/feature-cli-wireguard/src/lib/templates", + "glob": "*.tpl", + "output": "templates/wireguard" + }, + { + "input": "libs/domains/loadweaver/shared/feature-cli-vip/src/lib/templates", + "glob": "*.tpl", + "output": "templates/vip" + }, + { + "input": "libs/domains/loadweaver/shared/feature-cli-traefik/src/lib/templates", + "glob": "*.tpl", + "output": "templates/traefik" + }, + { + "input": "libs/domains/loadweaver/shared/feature-cli-routing/src/lib/templates", + "glob": "*.tpl", + "output": "templates/routing" + } + ], + "generatePackageJson": true, + "thirdParty": true, + "esbuildOptions": { + "sourcemap": true, + "loader": { + ".tpl": "text" + }, + "outExtension": { + ".js": ".js" + } + } + }, + "configurations": { + "development": {}, + "production": { + "esbuildOptions": { + "sourcemap": false, + "outExtension": { + ".js": ".js" + } + } + } + } + }, + "prune-lockfile": { + "dependsOn": ["build"], + "cache": true, + "executor": "@nx/js:prune-lockfile", + "outputs": [ + "{workspaceRoot}/dist/apps/loadweaver/cli-loadweaver/package.json", + "{workspaceRoot}/dist/apps/loadweaver/cli-loadweaver/package-lock.json" + ], + "options": { + "buildTarget": "build" + } + }, + "copy-workspace-modules": { + "dependsOn": ["build"], + "cache": true, + "outputs": [ + "{workspaceRoot}/dist/apps/loadweaver/cli-loadweaver/workspace_modules" + ], + "executor": "@nx/js:copy-workspace-modules", + "options": { + "buildTarget": "build" + } + }, + "prune": { + "dependsOn": ["prune-lockfile", "copy-workspace-modules"], + "executor": "nx:noop" + }, + "serve": { + "continuous": true, + "executor": "@nx/js:node", + "defaultConfiguration": "development", + "dependsOn": ["build"], + "options": { + "buildTarget": "loadweaver-cli-loadweaver:build", + "runBuildTargetDependencies": false + }, + "configurations": { + "development": { + "buildTarget": "loadweaver-cli-loadweaver:build:development" + }, + "production": { + "buildTarget": "loadweaver-cli-loadweaver:build:production" + } + } + }, + "test": { + "executor": "@nx/jest:jest", + "outputs": ["{workspaceRoot}/coverage/{projectRoot}"], + "options": { + "jestConfig": "apps/loadweaver/cli-loadweaver/jest.config.cts", + "passWithNoTests": true + } + }, + "run": { + "executor": "nx:run-commands", + "options": { + "command": "node dist/apps/loadweaver/cli-loadweaver/main.js" + }, + "dependsOn": [ + { + "target": "build" + } + ] + }, + "binary": { + "executor": "nx:run-commands", + "dependsOn": ["build"], + "outputs": [ + "{workspaceRoot}/dist/apps/loadweaver/cli-loadweaver/bin/loadweaver" + ], + "options": { + "commands": [ + "npx esbuild apps/loadweaver/cli-loadweaver/src/main.ts --bundle --platform=node --format=cjs --loader:.tpl=text --outfile=dist/apps/loadweaver/cli-loadweaver/bin/loadweaver.mjs --banner:js='#!/usr/bin/env node' --tsconfig=apps/loadweaver/cli-loadweaver/tsconfig.app.json", + "mv dist/apps/loadweaver/cli-loadweaver/bin/loadweaver.mjs dist/apps/loadweaver/cli-loadweaver/bin/loadweaver", + "chmod +x dist/apps/loadweaver/cli-loadweaver/bin/loadweaver" + ], + "parallel": false + } + } + } +} diff --git a/apps/loadweaver/cli-loadweaver/src/assets/.gitkeep b/apps/loadweaver/cli-loadweaver/src/assets/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/apps/loadweaver/cli-loadweaver/src/main.ts b/apps/loadweaver/cli-loadweaver/src/main.ts new file mode 100644 index 000000000..769b7d91d --- /dev/null +++ b/apps/loadweaver/cli-loadweaver/src/main.ts @@ -0,0 +1,8 @@ +import { createProgram } from '@forepath/loadweaver/shared/feature-cli/program'; + +const program = createProgram(); + +program.parseAsync(process.argv).catch((err: Error) => { + console.error(err.message); + process.exit(1); +}); diff --git a/apps/loadweaver/cli-loadweaver/tsconfig.app.json b/apps/loadweaver/cli-loadweaver/tsconfig.app.json new file mode 100644 index 000000000..37c6ca61e --- /dev/null +++ b/apps/loadweaver/cli-loadweaver/tsconfig.app.json @@ -0,0 +1,15 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "../../../dist/out-tsc", + "module": "commonjs", + "types": ["node"] + }, + "include": ["src/**/*.ts"], + "exclude": [ + "jest.config.ts", + "jest.config.cts", + "src/**/*.spec.ts", + "src/**/*.test.ts" + ] +} diff --git a/apps/loadweaver/cli-loadweaver/tsconfig.json b/apps/loadweaver/cli-loadweaver/tsconfig.json new file mode 100644 index 000000000..fdfa691bd --- /dev/null +++ b/apps/loadweaver/cli-loadweaver/tsconfig.json @@ -0,0 +1,16 @@ +{ + "extends": "../../../tsconfig.base.json", + "files": [], + "include": [], + "references": [ + { + "path": "./tsconfig.app.json" + }, + { + "path": "./tsconfig.spec.json" + } + ], + "compilerOptions": { + "esModuleInterop": true + } +} diff --git a/apps/loadweaver/cli-loadweaver/tsconfig.spec.json b/apps/loadweaver/cli-loadweaver/tsconfig.spec.json new file mode 100644 index 000000000..ddda10a3c --- /dev/null +++ b/apps/loadweaver/cli-loadweaver/tsconfig.spec.json @@ -0,0 +1,16 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "../../../dist/out-tsc", + "module": "commonjs", + "moduleResolution": "node10", + "types": ["jest", "node"] + }, + "include": [ + "jest.config.ts", + "jest.config.cts", + "src/**/*.test.ts", + "src/**/*.spec.ts", + "src/**/*.d.ts" + ] +} diff --git a/docs/loadweaver/README.md b/docs/loadweaver/README.md new file mode 100644 index 000000000..d6f1235b6 --- /dev/null +++ b/docs/loadweaver/README.md @@ -0,0 +1,23 @@ +# Loadweaver + +Loadweaver is a Node.js CLI for managing multi-node infrastructure with Docker Swarm, WireGuard, Ceph, Traefik, and floating VIP ingress. + +## Quick start + +```bash +npx nx run loadweaver-cli-loadweaver:binary +./dist/apps/loadweaver/cli-loadweaver/bin/loadweaver hello +./dist/apps/loadweaver/cli-loadweaver/bin/loadweaver config init-template ./loadweaver.yml +./dist/apps/loadweaver/cli-loadweaver/bin/loadweaver config validate --config ./loadweaver.yml +``` + +## Documentation + +- [Getting started](./getting-started.md) +- [Architecture overview](./architecture/system-overview.md) +- [CLI reference](./cli-reference.md) +- [Configuration](./configuration.md) +- [Workspace state](./workspace.md) +- [System requirements](./deployment/system-requirements.md) +- [Lab guide (3 VMs)](./deployment/lab-guide.md) +- [Production TLS (ACME / DNS)](./deployment/production-tls.md) diff --git a/docs/loadweaver/architecture/system-overview.md b/docs/loadweaver/architecture/system-overview.md new file mode 100644 index 000000000..c8474c098 --- /dev/null +++ b/docs/loadweaver/architecture/system-overview.md @@ -0,0 +1,22 @@ +# Architecture overview + +Loadweaver manages a multi-site cluster where: + +- **WireGuard** provides an L3 full mesh (`wg0`) between nodes +- **Docker Swarm** runs services with overlay networks advertised over WireGuard +- **Ceph/CephFS** backs shared storage mounted on every node +- **Traefik** serves ingress from a Swarm stack with shared config/certs on CephFS +- **VIP/keepalived** fronts Traefik for external traffic +- **BIRD (optional)** on hub nodes exchanges routes between sites and remote clusters over WireGuard + +See [architecture.mmd](../../libs/domains/loadweaver/shared/feature-cli/docs/architecture.mmd) for the general overview diagram. Focused breakdowns (CLI libraries, network layers, routing, orchestrator sequences) live alongside it under [`feature-cli/docs/`](../../libs/domains/loadweaver/shared/feature-cli/docs/). + +## Execution model + +The CLI runs from an operator workstation and converges remote nodes over SSH using idempotent commands. **Host bootstrap** installs Docker, WireGuard, keepalived, and cephadm before the stack is configured. Use `--dry-run` to inspect planned operations. + +The only prerequisite on the operator machine is SSH access to every node. + +## Future extensions + +Inter-cluster routing is available via optional `routing` configuration and BIRD on hub nodes. Each cluster keeps its own Swarm, Ceph, and WireGuard mesh; hub nodes peer with remote clusters over eBGP (and optional cross-cluster WireGuard tunnels). See [configuration.md](../configuration.md#advanced-routing-bird) and [inter-cluster routing](./deployment/inter-cluster-routing.md). diff --git a/docs/loadweaver/cli-reference.md b/docs/loadweaver/cli-reference.md new file mode 100644 index 000000000..2c9701032 --- /dev/null +++ b/docs/loadweaver/cli-reference.md @@ -0,0 +1,123 @@ +# CLI reference + +Top-level command: `loadweaver` + +Global flags: + +- `--config ` — configuration file (default `./loadweaver.yml`) +- `--env ` — merge a named profile overlay from the file +- `LOADWEAVER_CONFIG` — optional YAML/JSON in the environment, deep-merged over the file (and profile); env wins +- `--dry-run` — print planned commands; skip locks, persistence, and live drift probes +- `--verbose` — reserved; progress output is shown at the default log level +- `--debug` — include full SSH command traces +- `--yes` — skip drift confirmation and proceed (overwrite live); required for `cluster destroy`; allows destructive update actions +- `--accept-drift` — when drift is detected, refresh inventory from live, then continue (non-interactive skip) +- `--local` — restrict operations to the local node +- `--json` — machine-readable JSON for `cluster drift`, `cluster status`, `wireguard status`, `host status`, `diag ssh`, and dry-run step lists from `cluster init|update|destroy` + +## Commands + +| Group | Commands | +| ----------- | ------------------------------------------------------------------------------------------------------------------- | +| `hello` | Validate CLI wiring | +| `config` | `show`, `validate`, `init-template` | +| `host` | `bootstrap`, `verify`, `status` | +| `cluster` | `init`, `update`, `destroy`, `status`, `drift` | +| `node` | `join`, `leave`, `label` | +| `wireguard` | `init`, `add-peer`, `remove-peer`, `rotate-keys`, `rotation-status`, `rotate-if-due`, `rotation-schedule`, `status` | +| `swarm` | `init`, `join`, `reconcile-labels`, `network create`, `status` | +| `ssh` | Run a command on a node (` [command...]`) | +| `ceph` | `init`, `cephfs-create`, `cephfs-mount`, `osd-add`, `osd-reconcile`, `osd-remove`, `status` | +| `volume` | `create`, `list` | +| `traefik` | `deploy`, `update`, `destroy`, `status`, `verify-acme`, `acme-env init` | +| `vip` | `init`, `reconcile`, `status`, `verify-failover`, `destroy` | +| `routing` | `init`, `reconcile`, `status`, `destroy` | +| `diag` | `all`, `ping`, `ssh` | + +Run `loadweaver --help` for details and examples. + +## Progress output + +By default, Loadweaver prints: + +- Orchestrator steps (`Running step: …`) +- High-level service actions (bootstrap, WireGuard deploy, label reconcile, …) +- Per-node SSH progress: `→ node-a1 (root@203.0.113.10): ssh exec — apt-get update` + +Use `--debug` to print full SSH/SCP command lines. Use `--dry-run` to mark remote actions with `[dry-run]` without connecting. + +## Mutation safety + +Mutating commands acquire a workspace lock, complementary per-node host locks, and inspect remote drift (when local inventory exists): + +- Cluster: `init`, `update`, `destroy` +- Host: `bootstrap` +- Domain: `wireguard`, `swarm`, `ceph`, `volume`, `traefik`, `vip`, `routing`, `node` write operations + +Read-only commands (`status`, `drift`, `list`, `config show`, `ssh`) do not acquire locks. + +See [Workspace state](./workspace.md) for lock, inventory, drift, `--yes`, and `--accept-drift` details. + +## WireGuard key rotation + +Manual rotation: + +```bash +# Rotate all node keys and roll out updated peer configs +loadweaver wireguard rotate-keys + +# Rotate a single node +loadweaver wireguard rotate-keys node-a1 +``` + +Scheduled rotation (configure `wireguard.keyRotation` in `loadweaver.yml`): + +```bash +# Inspect key age and due/warning state +loadweaver wireguard rotation-status +loadweaver --json wireguard rotation-status + +# Exit codes for monitoring (when wireguard.keyRotation.enabled is true): +# 0 = all keys ok, 1 = at least one key due, 2 = warning window only + +# Rotate only overdue keys (for cron/systemd automation) +loadweaver --yes wireguard rotate-if-due + +# Print suggested cron/systemd timer snippets +loadweaver wireguard rotation-schedule +``` + +When `wireguard.keyRotation.enabled` is true, `cluster update` rotates overdue keys before other update steps. + +## Swarm label reconcile + +`swarm reconcile-labels` and `cluster update` (via `swarm.reconcile-labels`) add expected loadweaver labels and remove stale managed labels. + +Label encoding (Docker-compatible, supports multiple roles per node): + +- Roles: `loadweaver.role.=true` (for example `loadweaver.role.manager=true`) +- Routing hubs (when `routing.enabled`): `loadweaver.role.router=true` +- Sites: `loadweaver.site.=true` +- Legacy single-key labels (`loadweaver.role`, `loadweaver.site`) are removed during reconcile + +```bash +loadweaver swarm reconcile-labels +loadweaver swarm reconcile-labels node-a2 +``` + +`cluster destroy` runs `wg-quick down` on all nodes and removes `.loadweaver/wireguard/keys.json`. + +## Ad-hoc SSH + +Run a command on a cluster node using the same SSH target resolution as other Loadweaver commands (config `nodes..ssh` or defaults): + +```bash +loadweaver ssh node-a1 -- docker ps +loadweaver ssh node-a1 -- ls -la /mnt/cephfs +loadweaver --json ssh node-a1 -- docker ps +loadweaver --dry-run ssh node-a1 -- systemctl status keepalived +``` + +The remote exit code is propagated to the local process. Use `--json` for structured output (`nodeId`, `command`, `exitCode`, `stdout`, `stderr`, `dryRun`). + +`diag ssh` only verifies SSH connectivity and host readiness; use top-level `ssh` to run arbitrary commands. diff --git a/docs/loadweaver/configuration.md b/docs/loadweaver/configuration.md new file mode 100644 index 000000000..26da829c2 --- /dev/null +++ b/docs/loadweaver/configuration.md @@ -0,0 +1,261 @@ +# Configuration + +Loadweaver reads YAML/JSON from `loadweaver.yml` (override with `--config`). + +## Secrets and config overlays + +Do not commit passwords into the main config if you can avoid it. Prefer: + +1. **`LOADWEAVER_CONFIG` (recommended for selective secrets)** — YAML (or JSON) in the environment, deep-merged over the file. Env wins on conflicts; nested keys merge selectively (arrays are replaced wholesale). +2. **Gitignored local config** — `--config ./loadweaver.local.yml` +3. **Host-side secrets** — Traefik DNS tokens in `/etc/loadweaver/traefik-acme.env` (not in YAML) + +### `LOADWEAVER_CONFIG` + +Merge order: file → `--env` profile overlay → `LOADWEAVER_CONFIG` → schema validation. + +```bash +# Only override keepalived password; keep the rest from loadweaver.yml +export LOADWEAVER_CONFIG=' +vip: + authPass: your8ch +' + +loadweaver --config ./loadweaver.yml config show +loadweaver --config ./loadweaver.yml cluster update +``` + +Nested example (file `vip.address` / `vip.interface` remain; only `authPass` changes): + +```bash +export LOADWEAVER_CONFIG="$(cat <<'EOF' +vip: + authPass: prodsecret +EOF +)" +``` + +`config show` and `config validate` apply the same merge, so you can inspect the effective config before mutating. + +## Key sections + +- `cluster` — cluster name and primary manager node id +- `sites` — optional multi-site groupings of node ids (metadata/validation only) +- `nodes` — per-node hostname, public/private IP, WireGuard IP, roles, optional `osdDevice` for Ceph OSD scale-out, optional SSH overrides +- `ssh` — optional global SSH defaults (user, port, identity file, proxy jump, connection timeouts) +- `wireguard` — interface name, listen port, MTU +- `swarm` — advertise interface and overlay network list +- `ceph` — filesystem name, mount path, replication factor, Ceph release for cephadm (`release`, default `quincy`) +- `traefik` — image, network, deployment mode, optional ACME settings +- `vip` — Traefik floating IP and optional named L4 pools (keepalived + host HAProxy) +- `routing` — optional BIRD hub routing (disabled by default; iBGP within cluster, eBGP to remote clusters) +- `volumes` — Docker bind volumes mapped to CephFS subdirectories + +Generate a starter file: + +```bash +loadweaver config init-template ./loadweaver.yml +``` + +Validate schema and SSH settings: + +```bash +loadweaver config validate --config ./loadweaver.yml +``` + +Production profiles should define at least three manager nodes for Swarm quorum. + +## SSH access + +Loadweaver connects to nodes with the system `ssh` and `scp` binaries using key-based auth (`BatchMode=yes`). + +Global defaults: + +```yaml +ssh: + user: root + port: 22 + identityFile: ~/.ssh/loadweaver_ed25519 + proxyJump: bastion.example.com + connectTimeoutSeconds: 10 + serverAliveIntervalSeconds: 15 +``` + +Connection timeouts apply to every `ssh` and `scp` call Loadweaver makes. Defaults are **10** seconds for TCP connect (`ConnectTimeout`) and **15** seconds for keepalive probes (`ServerAliveInterval`). Set either value to **0** to disable that option and fall back to OpenSSH/system defaults. + +Per-node overrides (take precedence over global values): + +```yaml +nodes: + node-a1: + hostname: a1.example.com + publicIp: 203.0.113.10 + wireguardIp: 10.200.0.1 + roles: [manager] + sshUser: deploy + sshPort: 2222 + identityFile: ~/.ssh/node-a1 + proxyJump: jump-a.example.com +``` + +Host resolution order for SSH: `publicIp` → `privateIp` → `hostname`. + +### Validation + +`loadweaver config validate` and cluster prerequisite checks: + +- **Fail** if a configured `identityFile` path does not exist locally (supports `~/` expansion) +- **Warn** if a configured `proxyJump` host is unreachable (mutation may still be attempted) + +Ensure the operator workstation can reach nodes (directly or via jump host) before running converge commands. + +## Advanced routing (BIRD) + +By default Loadweaver uses static WireGuard mesh routes only. Enable BIRD on **routing hub nodes** when you need dynamic route exchange between sites or remote clusters. + +```yaml +routing: + enabled: true + localAsn: 64512 + clusterCidr: 10.200.0.0/24 # optional; derived from wireguardIp /24 when omitted + hubNodes: [node-a1, node-b1] # optional; defaults to first manager per site + exportWireguardSubnet: true + peers: + - name: staging + remoteAsn: 64513 + neighbor: 10.201.0.1 + importFilter: accept # accept | none + exportFilter: cluster # cluster | none + wireguardPeer: # optional cross-cluster tunnel on hubs (wg1) + publicKey: '' + endpoint: staging-hub.example.com:51821 + allowedIps: [10.201.0.0/24] + interface: wg1 + listenPort: 51821 + localAddress: 10.210.0.1/32 +``` + +When `routing.enabled` is false or the section is omitted, no BIRD package or configuration is applied. + +Hub nodes receive `bird2` during `host bootstrap`. BIRD is configured after `wireguard init` during `cluster init`. Routing hubs also receive the Swarm label `loadweaver.role.router=true` (alongside their configured roles) so placement constraints can target edge routers. + +### Protocol choice: BGP at cluster edges + +Inter-cluster and multi-site hub routing uses **BGP** (iBGP between hubs in one cluster, eBGP to remote clusters). BGP fits **edge peering** between separate administrative domains with explicit ASN and prefix policy. + +Do not use OSPF for this pattern: OSPF is an IGP for one contiguous routing domain, not for controlled advertisement between independent cluster edges. + +Set `connectTimeoutSeconds` / `serverAliveIntervalSeconds` under `ssh:` if hub nodes are reached through slow links. + +## Host bootstrap + +Loadweaver installs required packages on each node before configuring the stack: + +```bash +loadweaver host bootstrap # all nodes +loadweaver host bootstrap node-a1 # single node +loadweaver host verify +``` + +`cluster init` runs `host.bootstrap` automatically as its first step. + +Supported target OS: Debian 12 and Ubuntu 22.04+. + +Optional host settings: + +```yaml +host: + configureFirewall: true + aptProxy: http://proxy.example.com:8080 +``` + +Non-root SSH users are supported when passwordless `sudo` is available. + +## Ceph OSD scale-out + +Assign the `ceph-osd` role and an `osdDevice` on each node that should host an OSD. Loadweaver registers the host with cephadm and runs `ceph orch daemon add osd` from the primary manager during `cluster init`, `ceph osd-reconcile`, and `cluster update` when devices are added or changed. + +```yaml +nodes: + node-a2: + hostname: a2 + wireguardIp: 10.200.0.2 + roles: [worker, ceph-osd] + osdDevice: /dev/sdb +``` + +Manual addition remains available: `loadweaver ceph osd-add node-a2 /dev/sdb`. + +## Traefik ACME (production TLS) + +Full runbook: [production TLS](./deployment/production-tls.md) (DNS vs HTTP-01, token export on primary manager, verification). + +Use **DNS-01** for production with real DNS. Use **HTTP-01** only when the VIP or public IP is reachable on port 80 from the internet. + +```yaml +traefik: + acme: + email: ops@example.com + challengeType: dns + dnsProvider: cloudflare + storagePath: /letsencrypt/acme.json +vip: + address: 203.0.113.100/32 + interface: eth0 + authPass: your8ch # keepalived VRRP password (max 8 characters) + pools: + - name: postgres + address: 203.0.113.101/32 + healthCheck: + type: tcp + port: 5432 + listeners: + - port: 5432 + protocol: tcp + backends: + - type: node + nodeId: node-a1 + port: 5432 + - type: host + host: 10.200.0.50 + port: 5432 + - type: swarm + service: postgres + port: 5432 +``` + +`vip.address` remains the Traefik VIP (health check on local `:80`). Each pool has an independent VRRP instance and optional HAProxy listeners. Backends may target a configured node (WireGuard IP), a host IP, or Swarm task IPs resolved at apply time. Use `vip reconcile` (or `cluster update`) to refresh Swarm backends. + +Before `loadweaver traefik deploy`, persist credentials on the primary manager: + +```bash +loadweaver traefik acme-env init +# On primary: edit /etc/loadweaver/traefik-acme.env (chmod 600) +loadweaver traefik deploy +loadweaver traefik verify-acme +``` + +Or export in the current shell: `export CF_DNS_API_TOKEN='...'` + +`verify-acme` checks stack resolver args, DNS env var injection into the Traefik service, and `acme.json` inside the container. + +## VIP failover verification + +After `vip init`, confirm each configured address is held by exactly one node: + +```bash +loadweaver vip status +loadweaver --json vip status +loadweaver vip verify-failover +loadweaver vip verify-failover --pool postgres +loadweaver vip reconcile +``` + +To exercise failover (stops keepalived briefly on the current holder): + +```bash +loadweaver --yes vip verify-failover --simulate +loadweaver --yes vip verify-failover --simulate --address 203.0.113.101 +``` + +Operator-local state lives in `.loadweaver/` next to the config file. See [Workspace state](./workspace.md) for `state.json`, locks, WireGuard keys, and drift behavior. diff --git a/docs/loadweaver/deployment/inter-cluster-routing.md b/docs/loadweaver/deployment/inter-cluster-routing.md new file mode 100644 index 000000000..47c85564d --- /dev/null +++ b/docs/loadweaver/deployment/inter-cluster-routing.md @@ -0,0 +1,70 @@ +# Inter-cluster routing + +Use this guide when two or more independent Loadweaver clusters must exchange routes (for example production and staging, or two geographic regions with separate Swarm/Ceph stacks). + +## Topology + +Each cluster has its own `loadweaver.yml`, WireGuard full mesh, Swarm, and Ceph. Only **routing hub nodes** (typically one manager per site) run BIRD. + +Hub nodes in cluster A peer with hub nodes in cluster B using: + +1. **eBGP** over an existing L3 path (when hub WireGuard IPs are already reachable), or +2. **eBGP over a dedicated cross-cluster WireGuard tunnel** (`wg1` by default) configured under `routing.peers[].wireguardPeer`. + +Workers and non-hub nodes are unchanged; they continue using the intra-cluster WireGuard mesh. + +## Why BGP (not OSPF) + +Loadweaver uses **BGP** for hub-to-hub and inter-cluster routing because hubs sit at **administrative boundaries** between independent clusters or sites. BGP is designed for policy-controlled prefix exchange between autonomous systems (ASNs), optional multihop peering, and explicit import/export filters. + +**OSPF** is a link-state interior gateway protocol (IGP) for a single routing domain. It assumes shared trust and flooding within one area — a poor fit when you are connecting **edges** of separate Loadweaver clusters that should only advertise agreed prefixes (for example the WireGuard CIDR) to each other. + +## Configuration checklist + +### Cluster A (`localAsn: 64512`) + +```yaml +routing: + enabled: true + localAsn: 64512 + hubNodes: [node-a1] + exportWireguardSubnet: true + peers: + - name: cluster-b + remoteAsn: 64513 + neighbor: 10.201.0.1 + exportFilter: cluster + importFilter: accept + wireguardPeer: + publicKey: '' + endpoint: b-hub.example.com:51821 + allowedIps: [10.201.0.0/24] + interface: wg1 + listenPort: 51821 + localAddress: 10.210.0.1/32 +``` + +### Cluster B (`localAsn: 64513`) + +Mirror the peer entry pointing back at cluster A's hub WireGuard or cross-link address. Use a **different** `localAsn` and non-overlapping `clusterCidr` / `wireguardIp` ranges. + +## Converge order + +1. Bootstrap and converge each cluster independently (`cluster init`). +2. Exchange cross-cluster WireGuard public keys out of band (or read from `birdc`/hub after first dry-run). +3. Add `routing.peers` on both sides and run `loadweaver routing reconcile` (or `cluster update`). + +Verify on a hub: + +```bash +loadweaver --config ./loadweaver.yml routing status +loadweaver --json --config ./loadweaver.yml routing status +``` + +Look for `Established` BGP sessions and imported remote cluster prefixes in `birdc show route` output. + +## Safety notes + +- Keep `exportFilter: cluster` unless you intentionally advertise more than the local WireGuard CIDR. +- Do not overlap `clusterCidr` or `wireguardPeer.allowedIps` between clusters; `config validate` rejects overlaps. +- Cross-cluster tunnels are hub-only; they do not replace the per-cluster full mesh on `wg0`. diff --git a/docs/loadweaver/deployment/lab-guide.md b/docs/loadweaver/deployment/lab-guide.md new file mode 100644 index 000000000..78d0e897a --- /dev/null +++ b/docs/loadweaver/deployment/lab-guide.md @@ -0,0 +1,174 @@ +# Lab guide (3 VMs) + +Use this guide to validate Loadweaver against real Debian 12 or Ubuntu 22.04 VMs before production use. + +## VM sizing + +| Role | vCPU | RAM | Disk | +| ------------------ | ---- | ---- | ----------------------------- | +| Manager + Ceph MON | 2 | 8 GB | 40 GB OS + optional OSD disk | +| Worker + Ceph OSD | 2 | 8 GB | 40 GB OS + dedicated OSD disk | + +Minimum lab: **3 VMs** (3 managers for Swarm quorum, Ceph co-located for simplicity). + +## Network + +- Public or private IPs reachable from the operator workstation via SSH +- Allow outbound HTTPS for apt/Docker/Ceph repositories during bootstrap +- Loadweaver opens **ufw** rules when `host.configureFirewall: true` (default) + +Ports referenced during bootstrap: + +| Port | Service | +| -------------------------------- | ----------------------------------------------------------------------- | +| 22/tcp | SSH | +| 2377/tcp, 7946/tcp+udp, 4789/udp | Docker Swarm | +| 51820/udp | WireGuard (`wg0`, default) | +| 51821/udp | Cross-cluster WireGuard (`wg1`, when routing peers use `wireguardPeer`) | +| 179/tcp | BIRD BGP (routing hubs only, when `routing.enabled`) | +| 80/tcp, 443/tcp | Traefik | +| 112 | keepalived VRRP (when `vip` is configured) | + +## Operator setup + +1. Generate SSH key: `ssh-keygen -t ed25519 -f ~/.ssh/loadweaver_ed25519` +2. Install public key on each VM (`root` or sudo-capable user) +3. Build CLI: + +```bash +npx nx run loadweaver-cli-loadweaver:binary +``` + +## Configuration + +```bash +./dist/apps/loadweaver/cli-loadweaver/bin/loadweaver config init-template ./loadweaver-lab.yml +``` + +Edit: + +- `nodes.*.publicIp` / `privateIp` +- `nodes.*.wireguardIp` (unique /32 per node, e.g. `10.200.0.x`) +- `nodes.*.osdDevice` on OSD nodes (e.g. `/dev/sdb`) for automated Ceph scale-out +- `ssh.identityFile` if not using default agent keys +- `host.aptProxy` if VMs reach the internet via proxy +- `vip.authPass` (max 8 characters) when using a floating IP +- `traefik.acme` — use `challengeType: http` for simple labs; see [production TLS](./production-tls.md) for DNS-01 + +Validate: + +```bash +loadweaver config validate --config ./loadweaver-lab.yml +``` + +### Optional: inter-cluster routing lab + +Add a `routing` block and hub nodes to exercise BIRD/BGP. Hubs receive Swarm label `loadweaver.role.router=true`. See [inter-cluster routing](./inter-cluster-routing.md). + +## Converge sequence + +Dry-run first: + +```bash +loadweaver --dry-run --verbose --config ./loadweaver-lab.yml cluster init +loadweaver --json --dry-run --config ./loadweaver-lab.yml cluster init +``` + +Execute: + +```bash +loadweaver --config ./loadweaver-lab.yml cluster init +``` + +Verify: + +```bash +loadweaver --config ./loadweaver-lab.yml cluster status +loadweaver --config ./loadweaver-lab.yml cluster drift +loadweaver --json --config ./loadweaver-lab.yml host status +loadweaver --json --config ./loadweaver-lab.yml diag ssh node-a1 +loadweaver --json --config ./loadweaver-lab.yml wireguard status +loadweaver --config ./loadweaver-lab.yml ceph status +``` + +### VIP (when configured) + +```bash +loadweaver --json --config ./loadweaver-lab.yml vip status +loadweaver --config ./loadweaver-lab.yml vip verify-failover +loadweaver --config ./loadweaver-lab.yml vip verify-failover --pool postgres +loadweaver --yes --config ./loadweaver-lab.yml vip verify-failover --simulate +loadweaver --config ./loadweaver-lab.yml vip reconcile +``` + +Named `vip.pools` use independent VRRP instances. Listeners install host HAProxy; Swarm backends refresh on `vip reconcile` / `cluster update`. + +### Traefik ACME (DNS lab) + +On the primary manager, persist provider credentials then deploy: + +```bash +loadweaver --config ./loadweaver-lab.yml traefik acme-env init +# SSH to primary: edit /etc/loadweaver/traefik-acme.env (chmod 600), uncomment CF_DNS_API_TOKEN= +loadweaver --config ./loadweaver-lab.yml traefik deploy +loadweaver --json --config ./loadweaver-lab.yml traefik verify-acme +``` + +For HTTP-01 labs, ensure port 80 on the VIP is reachable before enabling ACME. See [production TLS](./production-tls.md). + +### Routing (when enabled) + +```bash +loadweaver --json --config ./loadweaver-lab.yml routing status +``` + +## Add a 4th node (incremental update) + +1. Add the node to `loadweaver-lab.yml` (include `roles: [worker, ceph-osd]` and `osdDevice` if it should host an OSD) +2. Run: + +```bash +loadweaver --dry-run --config ./loadweaver-lab.yml cluster update +loadweaver --config ./loadweaver-lab.yml cluster update +``` + +Expected planner steps include `host.bootstrap.`, `ceph.osd-add.` when `osdDevice` is set, and `swarm.join`. + +Manual OSD reconcile: + +```bash +loadweaver --config ./loadweaver-lab.yml ceph osd-reconcile +``` + +## Remove a node + +1. Remove the node from `loadweaver-lab.yml` +2. Run with confirmation: + +```bash +loadweaver --yes --config ./loadweaver-lab.yml cluster update +``` + +Expected steps: `ceph.osd-remove.` (when the node had OSDs), `node.leave.`, `wireguard.remove-peer.`. + +## Troubleshooting + +| Symptom | Check | +| ------------------------------ | ------------------------------------------------------------------------------------------- | +| Bootstrap fails on apt | VM outbound internet; set `host.aptProxy` | +| `sudo: a password is required` | Use root SSH or passwordless sudo for bootstrap user | +| Swarm join fails | `host bootstrap` completed; WireGuard mesh up (`loadweaver wireguard status`) | +| Ceph bootstrap fails | MON nodes have `cephadm`; sufficient RAM | +| OSD not added | `osdDevice` set; run `ceph osd-reconcile` or check `ceph orch device ls` on primary | +| VIP not held | `keepalived` active on all nodes; `vip.interface` matches NIC name | +| VIP split-brain | `loadweaver vip verify-failover`; check priorities in keepalived config | +| ACME DNS fails | `traefik acme-env init`; token in `/etc/loadweaver/traefik-acme.env`; `traefik verify-acme` | +| BGP session down | Port 179 between hubs; `routing status`; peer `neighbor` reachable | + +## Tear down + +```bash +loadweaver --yes --config ./loadweaver-lab.yml cluster destroy +``` + +Removes Traefik stack, Swarm membership, CephFS mounts, WireGuard interfaces, optional BIRD/routing config, and local `.loadweaver/` workspace state. diff --git a/docs/loadweaver/deployment/production-tls.md b/docs/loadweaver/deployment/production-tls.md new file mode 100644 index 000000000..1ee72953b --- /dev/null +++ b/docs/loadweaver/deployment/production-tls.md @@ -0,0 +1,113 @@ +# Production TLS with Traefik ACME + +Use this runbook when issuing Let's Encrypt certificates on a production Loadweaver cluster with real DNS. + +## Choose a challenge type + +| Challenge | When to use | Requirements | +| ----------------------------------- | --------------------------------------------------------------- | ---------------------------------------------------------------------------------- | +| **DNS-01** (`challengeType: dns`) | Production default; wildcards; VIP not on public :80 | DNS provider API token on primary manager | +| **HTTP-01** (`challengeType: http`) | Simple lab or when VIP/public IP serves port 80 to the internet | `vip.address` (or node public IP) reachable on **TCP 80** from the public internet | + +Use **DNS-01** unless you have confirmed HTTP-01 reachability (see below). + +## DNS-01 (recommended) + +### 1. Configure ACME in `loadweaver.yml` + +```yaml +traefik: + image: traefik:v3 + network: traefik-public + mode: global + acme: + email: ops@example.com + challengeType: dns + dnsProvider: cloudflare + storagePath: /letsencrypt/acme.json +``` + +Supported `dnsProvider` values: `cloudflare`, `route53`, `digitalocean`. + +### 2. Persist credentials on the primary manager + +**Option A — env file (recommended for production):** + +```bash +loadweaver traefik acme-env init +``` + +SSH to the primary manager, edit `/etc/loadweaver/traefik-acme.env` (mode `600`), uncomment and set values: + +```bash +# CF_DNS_API_TOKEN=your-scoped-token +``` + +`traefik deploy` sources this file automatically before `docker stack deploy`. + +**Option B — export in shell (quick tests):** + +```bash +export CF_DNS_API_TOKEN='your-token' +``` + +Use the same shell session (or re-export before each deploy). + +Custom path: set `traefik.acme.envFile` in `loadweaver.yml` (default `/etc/loadweaver/traefik-acme.env`). + +### 3. Deploy and verify + +```bash +loadweaver traefik deploy +loadweaver traefik verify-acme +loadweaver --json traefik verify-acme +``` + +`verify-acme` confirms: + +- Stack includes the expected DNS challenge resolver args +- Required env var **names** are present on the Traefik service (host → container injection) +- `acme.json` exists and is non-empty inside the running Traefik container + +### 4. Attach certificates to services + +Label Swarm services with Traefik router TLS settings pointing at resolver `le` (see Traefik v3 Swarm docs). + +## HTTP-01 (only when port 80 is public) + +```yaml +traefik: + acme: + email: ops@example.com + challengeType: http + storagePath: /letsencrypt/acme.json +vip: + address: 203.0.113.100/32 + interface: eth0 +``` + +Before relying on HTTP-01, confirm reachability from outside your network: + +```bash +curl -I http://203.0.113.100/ +# or your public hostname resolving to the VIP +``` + +If the VIP is private, behind NAT without port forwarding, or filtered by firewall, HTTP-01 will fail — switch to DNS-01. + +## Troubleshooting + +| Symptom | Likely cause | +| --------------------------------------------- | ----------------------------------------------------------------------------- | +| `Missing CF_DNS_API_TOKEN on primary manager` | Token not exported on primary before deploy | +| `missing DNS provider environment variables` | Stack deployed before token export; re-export and `loadweaver traefik update` | +| `acme.json missing or empty` | No certificate issued yet; check Traefik logs and DNS API permissions | +| HTTP-01 timeout | Port 80 not reachable on VIP from internet | + +## Related commands + +```bash +loadweaver --dry-run traefik deploy # inspect planned stack YAML +loadweaver --dry-run --json traefik verify-acme +loadweaver vip verify-failover # confirm VIP healthy before cutover +``` diff --git a/docs/loadweaver/deployment/system-requirements.md b/docs/loadweaver/deployment/system-requirements.md new file mode 100644 index 000000000..52d0ee212 --- /dev/null +++ b/docs/loadweaver/deployment/system-requirements.md @@ -0,0 +1,54 @@ +# System requirements + +Loadweaver bootstraps required host software over SSH. The operator workstation only needs SSH access to each node (key-based auth). + +## Operator workstation + +- SSH client +- Node.js runtime (to run the Loadweaver CLI) +- Network reachability to all cluster nodes (directly or via configured `proxyJump`) + +No Docker, WireGuard, or Ceph tools are required locally. + +## Target nodes + +- OS: **Debian 12** or **Ubuntu 22.04+** (host bootstrap uses `apt`) +- Root or sudo-capable SSH user (bootstrap scripts require root) +- CPU: 2 cores minimum (4+ recommended) +- RAM: 4 GB minimum (8+ GB for Ceph OSD / Traefik nodes) +- Disk: sufficient space for Docker images; dedicated disks recommended for Ceph OSDs +- Outbound internet access during bootstrap (Docker and Ceph apt repositories) + +## Installed automatically by `host bootstrap` + +| Component | When installed | +| -------------------------------------------- | ----------------------------------------- | +| Docker CE + compose plugin | All nodes | +| WireGuard + wireguard-tools | All nodes | +| keepalived | All nodes when `vip` is configured | +| haproxy | All nodes when any VIP pool has listeners | +| cephadm (+ repo) | Nodes with any `ceph-*` role | +| Base packages (`chrony`, `lvm2`, curl, etc.) | All nodes | + +Run standalone: + +```bash +loadweaver host bootstrap +loadweaver host verify +``` + +`cluster init` runs host bootstrap and verification before WireGuard/Swarm setup. + +## Network + +- Stable connectivity between nodes +- 1 Gbps minimum (10 Gbps recommended for Ceph-heavy workloads) + +## Ports + +- Swarm: 2377/tcp, 7946/tcp+udp, 4789/udp +- WireGuard: 51820/udp (configurable) +- Traefik: 80/tcp, 443/tcp +- VRRP/keepalived: protocol 112 +- VIP pool listeners: each configured listener TCP port (when firewall is enabled) +- Ceph: monitor/OSD ports per cephadm deployment diff --git a/docs/loadweaver/getting-started.md b/docs/loadweaver/getting-started.md new file mode 100644 index 000000000..67bd00717 --- /dev/null +++ b/docs/loadweaver/getting-started.md @@ -0,0 +1,38 @@ +# Getting started + +## Prerequisites + +- Node.js 18+ (to build/run the CLI) +- SSH key access as root (or a sudo-capable user) to every cluster node +- Debian 12 or Ubuntu 22.04+ on target nodes +- `loadweaver.yml` describing your cluster (see [configuration.md](./configuration.md)) + +No Docker, WireGuard, or Ceph tools are required on the operator workstation. + +## Build the CLI + +```bash +npx nx run loadweaver-cli-loadweaver:build +npx nx run loadweaver-cli-loadweaver:binary +``` + +## Bootstrap workflow + +1. Generate a template config: `loadweaver config init-template` +2. Edit node addresses, roles, WireGuard IPs, SSH settings, and VIP settings +3. Validate: `loadweaver config validate` +4. Dry-run full bootstrap: `loadweaver --dry-run cluster init` +5. Execute bootstrap: `loadweaver cluster init` + +`cluster init` installs host packages (`host bootstrap`), verifies software, then configures WireGuard → Swarm → Ceph → Traefik → VIP. + +You can also bootstrap hosts independently: + +```bash +loadweaver host bootstrap +loadweaver host verify +``` + +## Typical dependency order + +Host packages → WireGuard mesh → Docker Swarm → Ceph/CephFS → Docker volumes → overlay networks → Traefik → VIP diff --git a/docs/loadweaver/workspace.md b/docs/loadweaver/workspace.md new file mode 100644 index 000000000..932c9c83b --- /dev/null +++ b/docs/loadweaver/workspace.md @@ -0,0 +1,149 @@ +# Workspace state + +Loadweaver stores operator-local state beside your configuration file in a `.loadweaver/` directory. + +Example layout when `./loadweaver.yml` is the config path: + +```text +./loadweaver.yml +./.loadweaver/ + state.json + lock.json + wireguard/ + keys.json +``` + +Each managed node also stores: + +```text +/etc/loadweaver/inventory.json +/etc/loadweaver/lock.json # short-lived host mutex during mutations +``` + +## state.json + +Written after a successful guarded mutation (not in `--dry-run`). `cluster destroy` removes it. + +Contains: + +- Planner snapshot of the last converged configuration (node ids, Swarm label expectations, Traefik image/mode, overlay networks, volumes, VIP fingerprint/pools) +- `desired` — sanitized full `loadweaver.yml` snapshot (`vip.authPass` is redacted) +- `inventorySerial` — monotonic generation used to reconcile host inventory files +- `remoteFingerprint` — live cluster observations captured at persist time (Swarm membership, node labels, WireGuard peer counts, Traefik revision, per-node service health) + +Used by: + +- `cluster update` — incremental planner diffs against the last recorded config snapshot +- `cluster drift` — compares stored `remoteFingerprint` to the current cluster +- Mutation guards — warn before changes when drift is detected + +If `state.json` is missing, `cluster update` performs a full init path before recording state. Older state files without `desired` / `inventorySerial` remain valid; the next persist writes those fields. + +## lock.json + +Exclusive operational lock for mutating commands. + +- Acquired by `runGuardedMutation()` for all cluster and domain mutations (unless `--dry-run`) +- Records PID, operation name, and timestamp +- Stale locks (dead PID or older than 2 hours) are replaced automatically + +Prevents concurrent `cluster update`, `wireguard init`, `traefik deploy`, and similar commands from corrupting shared state on the **same workstation**. It is not a substitute for host inventory. + +## Host inventory and host locks + +`/etc/loadweaver/inventory.json` is the per-node copy of last applied inventory (serial, node identity, last operation, and that node's fingerprint). It is written after successful mutations and removed on `node leave` and `cluster destroy`. + +`/etc/loadweaver/lock.json` is a short-lived host mutex so two operator workstations cannot mutate the same node at once. Stale host locks (older than 2 hours) are replaced. Read-only commands (`ssh`, `diag`, `status`, `cluster drift`) do not take host locks. + +## WireGuard key rotation schedule + +Configure automatic rotation policy under `wireguard.keyRotation`: + +```yaml +wireguard: + keyRotation: + enabled: true + intervalDays: 90 + warnBeforeDays: 14 +``` + +Each key record in `wireguard/keys.json` stores `rotatedAt` (ISO timestamp). Keys without `rotatedAt` are treated as due when rotation is enabled. + +Commands: + +- `wireguard rotation-status` — show per-node age and due/warning state +- `wireguard rotate-if-due` — rotate only overdue keys (guarded mutation; use in cron) +- `wireguard rotation-schedule` — print suggested cron/systemd timer entries + +`cluster update` prepends `wireguard.rotate-if-due` when rotation is enabled and keys are overdue, even when no other config changes are pending. + +Suggested daily cron (non-interactive): + +```bash +0 3 * * * root loadweaver --config /etc/loadweaver/loadweaver.yml --yes wireguard rotate-if-due +``` + +Monitoring-only cron (alert without rotating): + +```bash +30 2 * * * root loadweaver --config /etc/loadweaver/loadweaver.yml wireguard rotation-status +# exit 1 = keys due, exit 2 = warning window +``` + +## wireguard/keys.json + +WireGuard private/public key pairs for each node id. + +- Created on first `wireguard init` or `cluster init` +- Mode `600` on write +- Removed by `cluster destroy` (via `wireguard teardownAndClearKeys`) +- Rotated with `wireguard rotate-keys [nodeId]` +- Automatically rotated when due via `wireguard rotate-if-due` or `cluster update` (when `wireguard.keyRotation.enabled` is true) +- Individual node keys removed on `wireguard remove-peer` + +Requires `wireguard-tools` (`wg`) on the operator workstation for non-dry-run key generation. + +## Drift detection + +Before mutating commands run, Loadweaver compares: + +- Stored `remoteFingerprint` to live probes +- Local `inventorySerial` to `/etc/loadweaver/inventory.json` on each node +- Host inventory fingerprints to live probes on that node + +Drift checks include: + +| Area | Examples | +| -------------- | ------------------------------------------------------------------------------------------------------ | +| Swarm | Membership changes, unknown hostnames, label changes | +| WireGuard | Interface down, peer count mismatch or change | +| Traefik | Stack removed, image change, service revision/replica summary change | +| VIP | keepalived active state and per-address VIP holder changes | +| Host inventory | Missing file, serial mismatch, cluster/node identity mismatch, live probe mismatch vs host fingerprint | + +Inspect drift without mutating: + +```bash +loadweaver cluster drift +``` + +When drift is found, mutating commands prompt: + +- `y` — proceed and overwrite live (keep stored inventory until the mutation persists a new snapshot) +- `s` — skip/accept drift: refresh local and host inventory from live, then continue the action +- `N` — abort + +`--yes` proceeds without a prompt (overwrite). `--accept-drift` takes the skip/refresh path without a prompt. Non-interactive sessions require one of those flags. + +`--dry-run` skips live drift collection and does not acquire locks or persist workspace or host inventory files. + +## --yes semantics + +| Scenario | Behavior | +| ---------------------------------- | -------------------------------------------------------------------------------- | +| Drift detected before mutation | `--yes` proceeds (overwrite); `--accept-drift` refreshes inventory then proceeds | +| `cluster destroy` | Required (unless `--dry-run`) | +| `cluster update` with node removal | Allows destructive planner actions | +| Non-TTY environments | `--yes` or `--accept-drift` required for drift prompts | + +`--yes` does not bypass schema validation or missing SSH identity file checks. diff --git a/graph/graph.json b/graph/graph.json index 5f582b9de..71d2e9125 100644 --- a/graph/graph.json +++ b/graph/graph.json @@ -1,6 +1,6 @@ { "version": 1, - "generatedAt": "2026-08-25T17:44:36.644Z", + "generatedAt": "2026-08-27T17:19:09.403Z", "nodes": [ { "id": "project:@forepath/test/mounted-plugin-fixture", @@ -325,6 +325,27 @@ "featureGroup": "feature" } }, + { + "id": "project:loadweaver-shared-feature-cli-wireguard", + "type": "lib", + "attrs": { + "name": "loadweaver-shared-feature-cli-wireguard", + "root": "libs/domains/loadweaver/shared/feature-cli-wireguard", + "tags": [ + "domain:loadweaver", + "scope:shared", + "type:feature" + ], + "type": "lib", + "targets": [ + "lint", + "test" + ], + "domain": "loadweaver", + "context": "shared", + "featureGroup": "feature" + } + }, { "id": "project:agenstra-backend-feature-agent-manager", "type": "lib", @@ -431,6 +452,69 @@ "featureGroup": "feature" } }, + { + "id": "project:loadweaver-shared-feature-cli-cluster", + "type": "lib", + "attrs": { + "name": "loadweaver-shared-feature-cli-cluster", + "root": "libs/domains/loadweaver/shared/feature-cli-cluster", + "tags": [ + "domain:loadweaver", + "scope:shared", + "type:feature" + ], + "type": "lib", + "targets": [ + "lint", + "test" + ], + "domain": "loadweaver", + "context": "shared", + "featureGroup": "feature" + } + }, + { + "id": "project:loadweaver-shared-feature-cli-routing", + "type": "lib", + "attrs": { + "name": "loadweaver-shared-feature-cli-routing", + "root": "libs/domains/loadweaver/shared/feature-cli-routing", + "tags": [ + "domain:loadweaver", + "scope:shared", + "type:feature" + ], + "type": "lib", + "targets": [ + "lint", + "test" + ], + "domain": "loadweaver", + "context": "shared", + "featureGroup": "feature" + } + }, + { + "id": "project:loadweaver-shared-feature-cli-traefik", + "type": "lib", + "attrs": { + "name": "loadweaver-shared-feature-cli-traefik", + "root": "libs/domains/loadweaver/shared/feature-cli-traefik", + "tags": [ + "domain:loadweaver", + "scope:shared", + "type:feature" + ], + "type": "lib", + "targets": [ + "lint", + "test" + ], + "domain": "loadweaver", + "context": "shared", + "featureGroup": "feature" + } + }, { "id": "project:shared-frontend-feature-notifications", "type": "lib", @@ -494,6 +578,27 @@ "featureGroup": "data-access" } }, + { + "id": "project:loadweaver-shared-feature-cli-volume", + "type": "lib", + "attrs": { + "name": "loadweaver-shared-feature-cli-volume", + "root": "libs/domains/loadweaver/shared/feature-cli-volume", + "tags": [ + "domain:loadweaver", + "scope:shared", + "type:feature" + ], + "type": "lib", + "targets": [ + "lint", + "test" + ], + "domain": "loadweaver", + "context": "shared", + "featureGroup": "feature" + } + }, { "id": "project:shared-backend-feature-notifications", "type": "lib", @@ -515,6 +620,27 @@ "featureGroup": "feature" } }, + { + "id": "project:loadweaver-shared-feature-cli-swarm", + "type": "lib", + "attrs": { + "name": "loadweaver-shared-feature-cli-swarm", + "root": "libs/domains/loadweaver/shared/feature-cli-swarm", + "tags": [ + "domain:loadweaver", + "scope:shared", + "type:feature" + ], + "type": "lib", + "targets": [ + "lint", + "test" + ], + "domain": "loadweaver", + "context": "shared", + "featureGroup": "feature" + } + }, { "id": "project:shared-backend-util-network-address", "type": "lib", @@ -641,6 +767,89 @@ "featureGroup": "data-access" } }, + { + "id": "project:loadweaver-shared-feature-cli-ceph", + "type": "lib", + "attrs": { + "name": "loadweaver-shared-feature-cli-ceph", + "root": "libs/domains/loadweaver/shared/feature-cli-ceph", + "tags": [ + "domain:loadweaver", + "scope:shared", + "type:feature" + ], + "type": "lib", + "targets": [ + "lint", + "test" + ], + "domain": "loadweaver", + "context": "shared", + "featureGroup": "feature" + } + }, + { + "id": "project:loadweaver-shared-feature-cli-diag", + "type": "lib", + "attrs": { + "name": "loadweaver-shared-feature-cli-diag", + "root": "libs/domains/loadweaver/shared/feature-cli-diag", + "tags": [ + "domain:loadweaver", + "scope:shared", + "type:feature" + ], + "type": "lib", + "targets": [ + "lint", + "test" + ], + "domain": "loadweaver", + "context": "shared", + "featureGroup": "feature" + } + }, + { + "id": "project:loadweaver-shared-feature-cli-host", + "type": "lib", + "attrs": { + "name": "loadweaver-shared-feature-cli-host", + "root": "libs/domains/loadweaver/shared/feature-cli-host", + "tags": [ + "domain:loadweaver", + "scope:shared", + "type:feature" + ], + "type": "lib", + "targets": [ + "test" + ], + "domain": "loadweaver", + "context": "shared", + "featureGroup": "feature" + } + }, + { + "id": "project:loadweaver-shared-feature-cli-node", + "type": "lib", + "attrs": { + "name": "loadweaver-shared-feature-cli-node", + "root": "libs/domains/loadweaver/shared/feature-cli-node", + "tags": [ + "domain:loadweaver", + "scope:shared", + "type:feature" + ], + "type": "lib", + "targets": [ + "lint", + "test" + ], + "domain": "loadweaver", + "context": "shared", + "featureGroup": "feature" + } + }, { "id": "project:shared-frontend-util-configuration", "type": "lib", @@ -683,6 +892,48 @@ "featureGroup": "util" } }, + { + "id": "project:loadweaver-shared-feature-cli-ssh", + "type": "lib", + "attrs": { + "name": "loadweaver-shared-feature-cli-ssh", + "root": "libs/domains/loadweaver/shared/feature-cli-ssh", + "tags": [ + "domain:loadweaver", + "scope:shared", + "type:feature" + ], + "type": "lib", + "targets": [ + "lint", + "test" + ], + "domain": "loadweaver", + "context": "shared", + "featureGroup": "feature" + } + }, + { + "id": "project:loadweaver-shared-feature-cli-vip", + "type": "lib", + "attrs": { + "name": "loadweaver-shared-feature-cli-vip", + "root": "libs/domains/loadweaver/shared/feature-cli-vip", + "tags": [ + "domain:loadweaver", + "scope:shared", + "type:feature" + ], + "type": "lib", + "targets": [ + "lint", + "test" + ], + "domain": "loadweaver", + "context": "shared", + "featureGroup": "feature" + } + }, { "id": "project:shared-backend-feature-monitoring", "type": "lib", @@ -704,6 +955,28 @@ "featureGroup": "feature" } }, + { + "id": "project:shared-shared-util-shell-executor", + "type": "lib", + "attrs": { + "name": "shared-shared-util-shell-executor", + "root": "libs/domains/shared/shared/util-shell-executor", + "tags": [ + "npm:private", + "domain:shared", + "scope:shared", + "type:util" + ], + "type": "lib", + "targets": [ + "lint", + "test" + ], + "domain": "shared", + "context": "shared", + "featureGroup": "util" + } + }, { "id": "project:shared-backend-util-http-context", "type": "lib", @@ -748,6 +1021,50 @@ "featureGroup": "util" } }, + { + "id": "project:shared-shared-util-config-loader", + "type": "lib", + "attrs": { + "name": "shared-shared-util-config-loader", + "root": "libs/domains/shared/shared/util-config-loader", + "tags": [ + "npm:private", + "domain:shared", + "scope:shared", + "type:util" + ], + "type": "lib", + "targets": [ + "lint", + "test" + ], + "domain": "shared", + "context": "shared", + "featureGroup": "util" + } + }, + { + "id": "project:loadweaver-shared-util-cli-core", + "type": "lib", + "attrs": { + "name": "loadweaver-shared-util-cli-core", + "root": "libs/domains/loadweaver/shared/util-cli-core", + "tags": [ + "npm:private", + "domain:loadweaver", + "scope:shared", + "type:util" + ], + "type": "lib", + "targets": [ + "lint", + "test" + ], + "domain": "loadweaver", + "context": "shared", + "featureGroup": "util" + } + }, { "id": "project:shared-backend-util-redis-cache", "type": "lib", @@ -874,6 +1191,27 @@ "featureGroup": "feature" } }, + { + "id": "project:loadweaver-shared-feature-cli", + "type": "lib", + "attrs": { + "name": "loadweaver-shared-feature-cli", + "root": "libs/domains/loadweaver/shared/feature-cli", + "tags": [ + "domain:loadweaver", + "scope:shared", + "type:feature" + ], + "type": "lib", + "targets": [ + "lint", + "test" + ], + "domain": "loadweaver", + "context": "shared", + "featureGroup": "feature" + } + }, { "id": "project:shared-frontend-feature-docs", "type": "lib", @@ -1174,6 +1512,28 @@ "featureGroup": "util" } }, + { + "id": "project:shared-shared-util-logger", + "type": "lib", + "attrs": { + "name": "shared-shared-util-logger", + "root": "libs/domains/shared/shared/util-logger", + "tags": [ + "npm:private", + "domain:shared", + "scope:shared", + "type:util" + ], + "type": "lib", + "targets": [ + "lint", + "test" + ], + "domain": "shared", + "context": "shared", + "featureGroup": "util" + } + }, { "id": "project:decabill-backend-billing-manager", "type": "app", @@ -1371,6 +1731,28 @@ "featureGroup": "app" } }, + { + "id": "project:shared-shared-util-ssh", + "type": "lib", + "attrs": { + "name": "shared-shared-util-ssh", + "root": "libs/domains/shared/shared/util-ssh", + "tags": [ + "npm:private", + "domain:shared", + "scope:shared", + "type:util" + ], + "type": "lib", + "targets": [ + "lint", + "test" + ], + "domain": "shared", + "context": "shared", + "featureGroup": "util" + } + }, { "id": "project:agenstra-frontend-landingpage", "type": "app", @@ -1493,6 +1875,34 @@ "featureGroup": "app" } }, + { + "id": "project:loadweaver-cli-loadweaver", + "type": "app", + "attrs": { + "name": "loadweaver-cli-loadweaver", + "root": "apps/loadweaver/cli-loadweaver", + "tags": [ + "type:app", + "scope:shared", + "domain:loadweaver" + ], + "type": "app", + "targets": [ + "lint", + "build", + "prune-lockfile", + "copy-workspace-modules", + "prune", + "serve", + "test", + "run", + "binary" + ], + "domain": "loadweaver", + "context": "shared", + "featureGroup": "app" + } + }, { "id": "project:agenstra-frontend-docs", "type": "app", @@ -1753,6 +2163,19 @@ "targets": [] } }, + { + "id": "tool:jest", + "type": "tool", + "attrs": { + "name": "jest", + "root": "tools/jest", + "tags": [ + "type:tool" + ], + "type": "tool", + "targets": [] + } + }, { "id": "package:@agentclientprotocol/sdk", "type": "package", @@ -2441,6 +2864,22 @@ "version": "0.2.84" } }, + { + "id": "package:commander", + "type": "package", + "attrs": { + "name": "commander", + "version": "14.0.3" + } + }, + { + "id": "package:yaml", + "type": "package", + "attrs": { + "name": "yaml", + "version": "2.9.0" + } + }, { "id": "package:mermaid", "type": "package", @@ -8920,6 +9359,24 @@ "projectName": "agenstra-frontend-feature-agent-console" } }, + { + "id": "file:libs/domains/loadweaver/shared/feature-cli-wireguard/README.md", + "type": "readme", + "attrs": { + "path": "libs/domains/loadweaver/shared/feature-cli-wireguard/README.md", + "languageOrKind": "md", + "projectName": "loadweaver-shared-feature-cli-wireguard" + } + }, + { + "id": "file:libs/domains/loadweaver/shared/feature-cli-wireguard/src/lib/wireguard.service.ts", + "type": "service", + "attrs": { + "path": "libs/domains/loadweaver/shared/feature-cli-wireguard/src/lib/wireguard.service.ts", + "languageOrKind": "ts", + "projectName": "loadweaver-shared-feature-cli-wireguard" + } + }, { "id": "file:libs/domains/agenstra/backend/feature-agent-manager/README.md", "type": "readme", @@ -10054,6 +10511,69 @@ "projectName": "forepath-frontend-feature-landingpage" } }, + { + "id": "file:libs/domains/loadweaver/shared/feature-cli-cluster/README.md", + "type": "readme", + "attrs": { + "path": "libs/domains/loadweaver/shared/feature-cli-cluster/README.md", + "languageOrKind": "md", + "projectName": "loadweaver-shared-feature-cli-cluster" + } + }, + { + "id": "file:libs/domains/loadweaver/shared/feature-cli-cluster/src/lib/node-inventory.service.ts", + "type": "service", + "attrs": { + "path": "libs/domains/loadweaver/shared/feature-cli-cluster/src/lib/node-inventory.service.ts", + "languageOrKind": "ts", + "projectName": "loadweaver-shared-feature-cli-cluster" + } + }, + { + "id": "file:libs/domains/loadweaver/shared/feature-cli-routing/src/lib/routing-cross-wg.service.ts", + "type": "service", + "attrs": { + "path": "libs/domains/loadweaver/shared/feature-cli-routing/src/lib/routing-cross-wg.service.ts", + "languageOrKind": "ts", + "projectName": "loadweaver-shared-feature-cli-routing" + } + }, + { + "id": "file:libs/domains/loadweaver/shared/feature-cli-routing/src/lib/routing.service.ts", + "type": "service", + "attrs": { + "path": "libs/domains/loadweaver/shared/feature-cli-routing/src/lib/routing.service.ts", + "languageOrKind": "ts", + "projectName": "loadweaver-shared-feature-cli-routing" + } + }, + { + "id": "file:libs/domains/loadweaver/shared/feature-cli-traefik/README.md", + "type": "readme", + "attrs": { + "path": "libs/domains/loadweaver/shared/feature-cli-traefik/README.md", + "languageOrKind": "md", + "projectName": "loadweaver-shared-feature-cli-traefik" + } + }, + { + "id": "file:libs/domains/loadweaver/shared/feature-cli-traefik/src/lib/traefik-acme-env.service.ts", + "type": "service", + "attrs": { + "path": "libs/domains/loadweaver/shared/feature-cli-traefik/src/lib/traefik-acme-env.service.ts", + "languageOrKind": "ts", + "projectName": "loadweaver-shared-feature-cli-traefik" + } + }, + { + "id": "file:libs/domains/loadweaver/shared/feature-cli-traefik/src/lib/traefik.service.ts", + "type": "service", + "attrs": { + "path": "libs/domains/loadweaver/shared/feature-cli-traefik/src/lib/traefik.service.ts", + "languageOrKind": "ts", + "projectName": "loadweaver-shared-feature-cli-traefik" + } + }, { "id": "file:libs/domains/shared/frontend/feature-notifications/README.md", "type": "readme", @@ -10133,6 +10653,24 @@ ] } }, + { + "id": "file:libs/domains/loadweaver/shared/feature-cli-volume/README.md", + "type": "readme", + "attrs": { + "path": "libs/domains/loadweaver/shared/feature-cli-volume/README.md", + "languageOrKind": "md", + "projectName": "loadweaver-shared-feature-cli-volume" + } + }, + { + "id": "file:libs/domains/loadweaver/shared/feature-cli-volume/src/lib/volume.service.ts", + "type": "service", + "attrs": { + "path": "libs/domains/loadweaver/shared/feature-cli-volume/src/lib/volume.service.ts", + "languageOrKind": "ts", + "projectName": "loadweaver-shared-feature-cli-volume" + } + }, { "id": "file:libs/domains/shared/backend/feature-notifications/README.md", "type": "readme", @@ -10277,6 +10815,24 @@ "projectName": "shared-backend-feature-notifications" } }, + { + "id": "file:libs/domains/loadweaver/shared/feature-cli-swarm/README.md", + "type": "readme", + "attrs": { + "path": "libs/domains/loadweaver/shared/feature-cli-swarm/README.md", + "languageOrKind": "md", + "projectName": "loadweaver-shared-feature-cli-swarm" + } + }, + { + "id": "file:libs/domains/loadweaver/shared/feature-cli-swarm/src/lib/swarm.service.ts", + "type": "service", + "attrs": { + "path": "libs/domains/loadweaver/shared/feature-cli-swarm/src/lib/swarm.service.ts", + "languageOrKind": "ts", + "projectName": "loadweaver-shared-feature-cli-swarm" + } + }, { "id": "file:libs/domains/shared/backend/util-network-address/README.md", "type": "readme", @@ -10392,6 +10948,69 @@ ] } }, + { + "id": "file:libs/domains/loadweaver/shared/feature-cli-ceph/README.md", + "type": "readme", + "attrs": { + "path": "libs/domains/loadweaver/shared/feature-cli-ceph/README.md", + "languageOrKind": "md", + "projectName": "loadweaver-shared-feature-cli-ceph" + } + }, + { + "id": "file:libs/domains/loadweaver/shared/feature-cli-ceph/src/lib/ceph.service.ts", + "type": "service", + "attrs": { + "path": "libs/domains/loadweaver/shared/feature-cli-ceph/src/lib/ceph.service.ts", + "languageOrKind": "ts", + "projectName": "loadweaver-shared-feature-cli-ceph" + } + }, + { + "id": "file:libs/domains/loadweaver/shared/feature-cli-diag/README.md", + "type": "readme", + "attrs": { + "path": "libs/domains/loadweaver/shared/feature-cli-diag/README.md", + "languageOrKind": "md", + "projectName": "loadweaver-shared-feature-cli-diag" + } + }, + { + "id": "file:libs/domains/loadweaver/shared/feature-cli-diag/src/lib/diag.service.ts", + "type": "service", + "attrs": { + "path": "libs/domains/loadweaver/shared/feature-cli-diag/src/lib/diag.service.ts", + "languageOrKind": "ts", + "projectName": "loadweaver-shared-feature-cli-diag" + } + }, + { + "id": "file:libs/domains/loadweaver/shared/feature-cli-host/src/lib/host.service.ts", + "type": "service", + "attrs": { + "path": "libs/domains/loadweaver/shared/feature-cli-host/src/lib/host.service.ts", + "languageOrKind": "ts", + "projectName": "loadweaver-shared-feature-cli-host" + } + }, + { + "id": "file:libs/domains/loadweaver/shared/feature-cli-node/README.md", + "type": "readme", + "attrs": { + "path": "libs/domains/loadweaver/shared/feature-cli-node/README.md", + "languageOrKind": "md", + "projectName": "loadweaver-shared-feature-cli-node" + } + }, + { + "id": "file:libs/domains/loadweaver/shared/feature-cli-node/src/lib/node.service.ts", + "type": "service", + "attrs": { + "path": "libs/domains/loadweaver/shared/feature-cli-node/src/lib/node.service.ts", + "languageOrKind": "ts", + "projectName": "loadweaver-shared-feature-cli-node" + } + }, { "id": "file:libs/domains/shared/frontend/util-configuration/README.md", "type": "readme", @@ -10419,6 +11038,69 @@ "projectName": "shared-shared-util-network-address" } }, + { + "id": "file:libs/domains/loadweaver/shared/feature-cli-ssh/README.md", + "type": "readme", + "attrs": { + "path": "libs/domains/loadweaver/shared/feature-cli-ssh/README.md", + "languageOrKind": "md", + "projectName": "loadweaver-shared-feature-cli-ssh" + } + }, + { + "id": "file:libs/domains/loadweaver/shared/feature-cli-ssh/src/lib/join-remote-command.service.ts", + "type": "service", + "attrs": { + "path": "libs/domains/loadweaver/shared/feature-cli-ssh/src/lib/join-remote-command.service.ts", + "languageOrKind": "ts", + "projectName": "loadweaver-shared-feature-cli-ssh" + } + }, + { + "id": "file:libs/domains/loadweaver/shared/feature-cli-ssh/src/lib/ssh.service.ts", + "type": "service", + "attrs": { + "path": "libs/domains/loadweaver/shared/feature-cli-ssh/src/lib/ssh.service.ts", + "languageOrKind": "ts", + "projectName": "loadweaver-shared-feature-cli-ssh" + } + }, + { + "id": "file:libs/domains/loadweaver/shared/feature-cli-vip/README.md", + "type": "readme", + "attrs": { + "path": "libs/domains/loadweaver/shared/feature-cli-vip/README.md", + "languageOrKind": "md", + "projectName": "loadweaver-shared-feature-cli-vip" + } + }, + { + "id": "file:libs/domains/loadweaver/shared/feature-cli-vip/src/lib/backend-resolution.service.ts", + "type": "service", + "attrs": { + "path": "libs/domains/loadweaver/shared/feature-cli-vip/src/lib/backend-resolution.service.ts", + "languageOrKind": "ts", + "projectName": "loadweaver-shared-feature-cli-vip" + } + }, + { + "id": "file:libs/domains/loadweaver/shared/feature-cli-vip/src/lib/vip-inspection.service.ts", + "type": "service", + "attrs": { + "path": "libs/domains/loadweaver/shared/feature-cli-vip/src/lib/vip-inspection.service.ts", + "languageOrKind": "ts", + "projectName": "loadweaver-shared-feature-cli-vip" + } + }, + { + "id": "file:libs/domains/loadweaver/shared/feature-cli-vip/src/lib/vip.service.ts", + "type": "service", + "attrs": { + "path": "libs/domains/loadweaver/shared/feature-cli-vip/src/lib/vip.service.ts", + "languageOrKind": "ts", + "projectName": "loadweaver-shared-feature-cli-vip" + } + }, { "id": "file:libs/domains/shared/backend/feature-monitoring/README.md", "type": "readme", @@ -10446,6 +11128,15 @@ "projectName": "shared-backend-feature-monitoring" } }, + { + "id": "file:libs/domains/shared/shared/util-shell-executor/README.md", + "type": "readme", + "attrs": { + "path": "libs/domains/shared/shared/util-shell-executor/README.md", + "languageOrKind": "md", + "projectName": "shared-shared-util-shell-executor" + } + }, { "id": "file:libs/domains/shared/backend/util-http-context/README.md", "type": "readme", @@ -10500,6 +11191,123 @@ "projectName": "shared-frontend-util-docs-parser" } }, + { + "id": "file:libs/domains/shared/shared/util-config-loader/README.md", + "type": "readme", + "attrs": { + "path": "libs/domains/shared/shared/util-config-loader/README.md", + "languageOrKind": "md", + "projectName": "shared-shared-util-config-loader" + } + }, + { + "id": "file:libs/domains/loadweaver/shared/util-cli-core/README.md", + "type": "readme", + "attrs": { + "path": "libs/domains/loadweaver/shared/util-cli-core/README.md", + "languageOrKind": "md", + "projectName": "loadweaver-shared-util-cli-core" + } + }, + { + "id": "file:libs/domains/loadweaver/shared/util-cli-core/src/lib/services/derive-cluster-cidr.service.ts", + "type": "service", + "attrs": { + "path": "libs/domains/loadweaver/shared/util-cli-core/src/lib/services/derive-cluster-cidr.service.ts", + "languageOrKind": "ts", + "projectName": "loadweaver-shared-util-cli-core" + } + }, + { + "id": "file:libs/domains/loadweaver/shared/util-cli-core/src/lib/services/derive-osd-devices.service.ts", + "type": "service", + "attrs": { + "path": "libs/domains/loadweaver/shared/util-cli-core/src/lib/services/derive-osd-devices.service.ts", + "languageOrKind": "ts", + "projectName": "loadweaver-shared-util-cli-core" + } + }, + { + "id": "file:libs/domains/loadweaver/shared/util-cli-core/src/lib/services/derive-vip-state.service.ts", + "type": "service", + "attrs": { + "path": "libs/domains/loadweaver/shared/util-cli-core/src/lib/services/derive-vip-state.service.ts", + "languageOrKind": "ts", + "projectName": "loadweaver-shared-util-cli-core" + } + }, + { + "id": "file:libs/domains/loadweaver/shared/util-cli-core/src/lib/services/expected-swarm-labels.service.ts", + "type": "service", + "attrs": { + "path": "libs/domains/loadweaver/shared/util-cli-core/src/lib/services/expected-swarm-labels.service.ts", + "languageOrKind": "ts", + "projectName": "loadweaver-shared-util-cli-core" + } + }, + { + "id": "file:libs/domains/loadweaver/shared/util-cli-core/src/lib/services/node-registry.service.ts", + "type": "service", + "attrs": { + "path": "libs/domains/loadweaver/shared/util-cli-core/src/lib/services/node-registry.service.ts", + "languageOrKind": "ts", + "projectName": "loadweaver-shared-util-cli-core" + } + }, + { + "id": "file:libs/domains/loadweaver/shared/util-cli-core/src/lib/services/prerequisite.service.ts", + "type": "service", + "attrs": { + "path": "libs/domains/loadweaver/shared/util-cli-core/src/lib/services/prerequisite.service.ts", + "languageOrKind": "ts", + "projectName": "loadweaver-shared-util-cli-core" + } + }, + { + "id": "file:libs/domains/loadweaver/shared/util-cli-core/src/lib/services/remote-exec.service.ts", + "type": "service", + "attrs": { + "path": "libs/domains/loadweaver/shared/util-cli-core/src/lib/services/remote-exec.service.ts", + "languageOrKind": "ts", + "projectName": "loadweaver-shared-util-cli-core" + } + }, + { + "id": "file:libs/domains/loadweaver/shared/util-cli-core/src/lib/services/resolve-routing-hubs.service.ts", + "type": "service", + "attrs": { + "path": "libs/domains/loadweaver/shared/util-cli-core/src/lib/services/resolve-routing-hubs.service.ts", + "languageOrKind": "ts", + "projectName": "loadweaver-shared-util-cli-core" + } + }, + { + "id": "file:libs/domains/loadweaver/shared/util-cli-core/src/lib/services/routing-state.service.ts", + "type": "service", + "attrs": { + "path": "libs/domains/loadweaver/shared/util-cli-core/src/lib/services/routing-state.service.ts", + "languageOrKind": "ts", + "projectName": "loadweaver-shared-util-cli-core" + } + }, + { + "id": "file:libs/domains/loadweaver/shared/util-cli-core/src/lib/services/ssh-config-validation.service.ts", + "type": "service", + "attrs": { + "path": "libs/domains/loadweaver/shared/util-cli-core/src/lib/services/ssh-config-validation.service.ts", + "languageOrKind": "ts", + "projectName": "loadweaver-shared-util-cli-core" + } + }, + { + "id": "file:libs/domains/loadweaver/shared/util-cli-core/src/lib/services/ssh-target.service.ts", + "type": "service", + "attrs": { + "path": "libs/domains/loadweaver/shared/util-cli-core/src/lib/services/ssh-target.service.ts", + "languageOrKind": "ts", + "projectName": "loadweaver-shared-util-cli-core" + } + }, { "id": "file:libs/domains/shared/backend/util-redis-cache/README.md", "type": "readme", @@ -11022,6 +11830,96 @@ "projectName": "identity-backend-feature-auth" } }, + { + "id": "file:libs/domains/loadweaver/shared/feature-cli/README.md", + "type": "readme", + "attrs": { + "path": "libs/domains/loadweaver/shared/feature-cli/README.md", + "languageOrKind": "md", + "projectName": "loadweaver-shared-feature-cli" + } + }, + { + "id": "file:libs/domains/loadweaver/shared/feature-cli/docs/architecture.mmd", + "type": "diagram", + "attrs": { + "path": "libs/domains/loadweaver/shared/feature-cli/docs/architecture.mmd", + "languageOrKind": "mmd", + "projectName": "loadweaver-shared-feature-cli" + } + }, + { + "id": "file:libs/domains/loadweaver/shared/feature-cli/docs/cli-overview.mmd", + "type": "diagram", + "attrs": { + "path": "libs/domains/loadweaver/shared/feature-cli/docs/cli-overview.mmd", + "languageOrKind": "mmd", + "projectName": "loadweaver-shared-feature-cli" + } + }, + { + "id": "file:libs/domains/loadweaver/shared/feature-cli/docs/cluster-init-sequence.mmd", + "type": "diagram", + "attrs": { + "path": "libs/domains/loadweaver/shared/feature-cli/docs/cluster-init-sequence.mmd", + "languageOrKind": "mmd", + "projectName": "loadweaver-shared-feature-cli" + } + }, + { + "id": "file:libs/domains/loadweaver/shared/feature-cli/docs/cluster-topology.mmd", + "type": "diagram", + "attrs": { + "path": "libs/domains/loadweaver/shared/feature-cli/docs/cluster-topology.mmd", + "languageOrKind": "mmd", + "projectName": "loadweaver-shared-feature-cli" + } + }, + { + "id": "file:libs/domains/loadweaver/shared/feature-cli/docs/cluster-update-planner.mmd", + "type": "diagram", + "attrs": { + "path": "libs/domains/loadweaver/shared/feature-cli/docs/cluster-update-planner.mmd", + "languageOrKind": "mmd", + "projectName": "loadweaver-shared-feature-cli" + } + }, + { + "id": "file:libs/domains/loadweaver/shared/feature-cli/docs/network-layers.mmd", + "type": "diagram", + "attrs": { + "path": "libs/domains/loadweaver/shared/feature-cli/docs/network-layers.mmd", + "languageOrKind": "mmd", + "projectName": "loadweaver-shared-feature-cli" + } + }, + { + "id": "file:libs/domains/loadweaver/shared/feature-cli/docs/routing-architecture.mmd", + "type": "diagram", + "attrs": { + "path": "libs/domains/loadweaver/shared/feature-cli/docs/routing-architecture.mmd", + "languageOrKind": "mmd", + "projectName": "loadweaver-shared-feature-cli" + } + }, + { + "id": "file:libs/domains/loadweaver/shared/feature-cli/docs/routing-inter-cluster.mmd", + "type": "diagram", + "attrs": { + "path": "libs/domains/loadweaver/shared/feature-cli/docs/routing-inter-cluster.mmd", + "languageOrKind": "mmd", + "projectName": "loadweaver-shared-feature-cli" + } + }, + { + "id": "file:libs/domains/loadweaver/shared/feature-cli/docs/workspace-state.mmd", + "type": "diagram", + "attrs": { + "path": "libs/domains/loadweaver/shared/feature-cli/docs/workspace-state.mmd", + "languageOrKind": "mmd", + "projectName": "loadweaver-shared-feature-cli" + } + }, { "id": "file:libs/domains/shared/frontend/feature-docs/README.md", "type": "readme", @@ -11310,6 +12208,15 @@ "projectName": "shared-frontend-util-meta" } }, + { + "id": "file:libs/domains/shared/shared/util-logger/README.md", + "type": "readme", + "attrs": { + "path": "libs/domains/shared/shared/util-logger/README.md", + "languageOrKind": "md", + "projectName": "shared-shared-util-logger" + } + }, { "id": "file:apps/decabill/backend-billing-manager/src/app/app.module.ts", "type": "module", @@ -11418,6 +12325,15 @@ "projectName": "shared-platform-authentication" } }, + { + "id": "file:libs/domains/shared/shared/util-ssh/README.md", + "type": "readme", + "attrs": { + "path": "libs/domains/shared/shared/util-ssh/README.md", + "languageOrKind": "md", + "projectName": "shared-shared-util-ssh" + } + }, { "id": "file:apps/agenstra/frontend-landingpage/src/app/viewport-scroller-offset.service.ts", "type": "service", @@ -12919,6 +13835,86 @@ "languageOrKind": "md" } }, + { + "id": "file:docs/loadweaver/README.md", + "type": "doc", + "attrs": { + "path": "docs/loadweaver/README.md", + "languageOrKind": "md" + } + }, + { + "id": "file:docs/loadweaver/architecture/system-overview.md", + "type": "doc", + "attrs": { + "path": "docs/loadweaver/architecture/system-overview.md", + "languageOrKind": "md" + } + }, + { + "id": "file:docs/loadweaver/cli-reference.md", + "type": "doc", + "attrs": { + "path": "docs/loadweaver/cli-reference.md", + "languageOrKind": "md" + } + }, + { + "id": "file:docs/loadweaver/configuration.md", + "type": "doc", + "attrs": { + "path": "docs/loadweaver/configuration.md", + "languageOrKind": "md" + } + }, + { + "id": "file:docs/loadweaver/deployment/inter-cluster-routing.md", + "type": "doc", + "attrs": { + "path": "docs/loadweaver/deployment/inter-cluster-routing.md", + "languageOrKind": "md" + } + }, + { + "id": "file:docs/loadweaver/deployment/lab-guide.md", + "type": "doc", + "attrs": { + "path": "docs/loadweaver/deployment/lab-guide.md", + "languageOrKind": "md" + } + }, + { + "id": "file:docs/loadweaver/deployment/production-tls.md", + "type": "doc", + "attrs": { + "path": "docs/loadweaver/deployment/production-tls.md", + "languageOrKind": "md" + } + }, + { + "id": "file:docs/loadweaver/deployment/system-requirements.md", + "type": "doc", + "attrs": { + "path": "docs/loadweaver/deployment/system-requirements.md", + "languageOrKind": "md" + } + }, + { + "id": "file:docs/loadweaver/getting-started.md", + "type": "doc", + "attrs": { + "path": "docs/loadweaver/getting-started.md", + "languageOrKind": "md" + } + }, + { + "id": "file:docs/loadweaver/workspace.md", + "type": "doc", + "attrs": { + "path": "docs/loadweaver/workspace.md", + "languageOrKind": "md" + } + }, { "id": "file:AGENTS.md", "type": "readme", @@ -29816,6 +30812,736 @@ "domain": "forepath" } }, + { + "id": "concept:loadweaver-loadweaver", + "type": "concept", + "attrs": { + "title": "Loadweaver", + "docPath": "docs/loadweaver/README.md", + "sectionAnchor": "loadweaver", + "domain": "loadweaver" + } + }, + { + "id": "concept:loadweaver-quick-start", + "type": "concept", + "attrs": { + "title": "Quick start", + "docPath": "docs/loadweaver/README.md", + "sectionAnchor": "quick-start", + "domain": "loadweaver" + } + }, + { + "id": "concept:loadweaver-documentation", + "type": "concept", + "attrs": { + "title": "Documentation", + "docPath": "docs/loadweaver/README.md", + "sectionAnchor": "documentation", + "domain": "loadweaver" + } + }, + { + "id": "concept:loadweaver-architecture-overview", + "type": "concept", + "attrs": { + "title": "Architecture overview", + "docPath": "docs/loadweaver/architecture/system-overview.md", + "sectionAnchor": "architecture-overview", + "domain": "loadweaver" + } + }, + { + "id": "concept:loadweaver-execution-model", + "type": "concept", + "attrs": { + "title": "Execution model", + "docPath": "docs/loadweaver/architecture/system-overview.md", + "sectionAnchor": "execution-model", + "domain": "loadweaver" + } + }, + { + "id": "concept:loadweaver-future-extensions", + "type": "concept", + "attrs": { + "title": "Future extensions", + "docPath": "docs/loadweaver/architecture/system-overview.md", + "sectionAnchor": "future-extensions", + "domain": "loadweaver" + } + }, + { + "id": "concept:loadweaver-cli-reference", + "type": "concept", + "attrs": { + "title": "CLI reference", + "docPath": "docs/loadweaver/cli-reference.md", + "sectionAnchor": "cli-reference", + "domain": "loadweaver" + } + }, + { + "id": "concept:loadweaver-commands", + "type": "concept", + "attrs": { + "title": "Commands", + "docPath": "docs/loadweaver/cli-reference.md", + "sectionAnchor": "commands", + "domain": "loadweaver" + } + }, + { + "id": "concept:loadweaver-progress-output", + "type": "concept", + "attrs": { + "title": "Progress output", + "docPath": "docs/loadweaver/cli-reference.md", + "sectionAnchor": "progress-output", + "domain": "loadweaver" + } + }, + { + "id": "concept:loadweaver-mutation-safety", + "type": "concept", + "attrs": { + "title": "Mutation safety", + "docPath": "docs/loadweaver/cli-reference.md", + "sectionAnchor": "mutation-safety", + "domain": "loadweaver" + } + }, + { + "id": "concept:loadweaver-wireguard-key-rotation", + "type": "concept", + "attrs": { + "title": "WireGuard key rotation", + "docPath": "docs/loadweaver/cli-reference.md", + "sectionAnchor": "wireguard-key-rotation", + "domain": "loadweaver" + } + }, + { + "id": "concept:loadweaver-rotate-all-node-keys-and-roll-out-updated-peer-configs", + "type": "concept", + "attrs": { + "title": "Rotate all node keys and roll out updated peer configs", + "docPath": "docs/loadweaver/cli-reference.md", + "sectionAnchor": "rotate-all-node-keys-and-roll-out-updated-peer-configs", + "domain": "loadweaver" + } + }, + { + "id": "concept:loadweaver-rotate-a-single-node", + "type": "concept", + "attrs": { + "title": "Rotate a single node", + "docPath": "docs/loadweaver/cli-reference.md", + "sectionAnchor": "rotate-a-single-node", + "domain": "loadweaver" + } + }, + { + "id": "concept:loadweaver-inspect-key-age-and-due-warning-state", + "type": "concept", + "attrs": { + "title": "Inspect key age and due/warning state", + "docPath": "docs/loadweaver/cli-reference.md", + "sectionAnchor": "inspect-key-age-and-due-warning-state", + "domain": "loadweaver" + } + }, + { + "id": "concept:loadweaver-exit-codes-for-monitoring-when-wireguard-keyrotation-enabled-is-true", + "type": "concept", + "attrs": { + "title": "Exit codes for monitoring (when wireguard.keyRotation.enabled is true):", + "docPath": "docs/loadweaver/cli-reference.md", + "sectionAnchor": "exit-codes-for-monitoring-when-wireguard-keyrotation-enabled-is-true", + "domain": "loadweaver" + } + }, + { + "id": "concept:loadweaver-0-all-keys-ok-1-at-least-one-key-due-2-warning-window-only", + "type": "concept", + "attrs": { + "title": "0 = all keys ok, 1 = at least one key due, 2 = warning window only", + "docPath": "docs/loadweaver/cli-reference.md", + "sectionAnchor": "0-all-keys-ok-1-at-least-one-key-due-2-warning-window-only", + "domain": "loadweaver" + } + }, + { + "id": "concept:loadweaver-rotate-only-overdue-keys-for-cron-systemd-automation", + "type": "concept", + "attrs": { + "title": "Rotate only overdue keys (for cron/systemd automation)", + "docPath": "docs/loadweaver/cli-reference.md", + "sectionAnchor": "rotate-only-overdue-keys-for-cron-systemd-automation", + "domain": "loadweaver" + } + }, + { + "id": "concept:loadweaver-print-suggested-cron-systemd-timer-snippets", + "type": "concept", + "attrs": { + "title": "Print suggested cron/systemd timer snippets", + "docPath": "docs/loadweaver/cli-reference.md", + "sectionAnchor": "print-suggested-cron-systemd-timer-snippets", + "domain": "loadweaver" + } + }, + { + "id": "concept:loadweaver-swarm-label-reconcile", + "type": "concept", + "attrs": { + "title": "Swarm label reconcile", + "docPath": "docs/loadweaver/cli-reference.md", + "sectionAnchor": "swarm-label-reconcile", + "domain": "loadweaver" + } + }, + { + "id": "concept:loadweaver-ad-hoc-ssh", + "type": "concept", + "attrs": { + "title": "Ad-hoc SSH", + "docPath": "docs/loadweaver/cli-reference.md", + "sectionAnchor": "ad-hoc-ssh", + "domain": "loadweaver" + } + }, + { + "id": "concept:loadweaver-configuration", + "type": "concept", + "attrs": { + "title": "Configuration", + "docPath": "docs/loadweaver/configuration.md", + "sectionAnchor": "configuration", + "domain": "loadweaver" + } + }, + { + "id": "concept:loadweaver-secrets-and-config-overlays", + "type": "concept", + "attrs": { + "title": "Secrets and config overlays", + "docPath": "docs/loadweaver/configuration.md", + "sectionAnchor": "secrets-and-config-overlays", + "domain": "loadweaver" + } + }, + { + "id": "concept:loadweaver-only-override-keepalived-password-keep-the-rest-from-loadweaver-yml", + "type": "concept", + "attrs": { + "title": "Only override keepalived password; keep the rest from loadweaver.yml", + "docPath": "docs/loadweaver/configuration.md", + "sectionAnchor": "only-override-keepalived-password-keep-the-rest-from-loadweaver-yml", + "domain": "loadweaver" + } + }, + { + "id": "concept:loadweaver-key-sections", + "type": "concept", + "attrs": { + "title": "Key sections", + "docPath": "docs/loadweaver/configuration.md", + "sectionAnchor": "key-sections", + "domain": "loadweaver" + } + }, + { + "id": "concept:loadweaver-ssh-access", + "type": "concept", + "attrs": { + "title": "SSH access", + "docPath": "docs/loadweaver/configuration.md", + "sectionAnchor": "ssh-access", + "domain": "loadweaver" + } + }, + { + "id": "concept:loadweaver-advanced-routing-bird", + "type": "concept", + "attrs": { + "title": "Advanced routing (BIRD)", + "docPath": "docs/loadweaver/configuration.md", + "sectionAnchor": "advanced-routing-bird", + "domain": "loadweaver" + } + }, + { + "id": "concept:loadweaver-host-bootstrap", + "type": "concept", + "attrs": { + "title": "Host bootstrap", + "docPath": "docs/loadweaver/configuration.md", + "sectionAnchor": "host-bootstrap", + "domain": "loadweaver" + } + }, + { + "id": "concept:loadweaver-ceph-osd-scale-out", + "type": "concept", + "attrs": { + "title": "Ceph OSD scale-out", + "docPath": "docs/loadweaver/configuration.md", + "sectionAnchor": "ceph-osd-scale-out", + "domain": "loadweaver" + } + }, + { + "id": "concept:loadweaver-traefik-acme-production-tls", + "type": "concept", + "attrs": { + "title": "Traefik ACME (production TLS)", + "docPath": "docs/loadweaver/configuration.md", + "sectionAnchor": "traefik-acme-production-tls", + "domain": "loadweaver" + } + }, + { + "id": "concept:loadweaver-on-primary-edit-etc-loadweaver-traefik-acme-env-chmod-600", + "type": "concept", + "attrs": { + "title": "On primary: edit /etc/loadweaver/traefik-acme.env (chmod 600)", + "docPath": "docs/loadweaver/configuration.md", + "sectionAnchor": "on-primary-edit-etc-loadweaver-traefik-acme-env-chmod-600", + "domain": "loadweaver" + } + }, + { + "id": "concept:loadweaver-vip-failover-verification", + "type": "concept", + "attrs": { + "title": "VIP failover verification", + "docPath": "docs/loadweaver/configuration.md", + "sectionAnchor": "vip-failover-verification", + "domain": "loadweaver" + } + }, + { + "id": "concept:loadweaver-inter-cluster-routing", + "type": "concept", + "attrs": { + "title": "Inter-cluster routing", + "docPath": "docs/loadweaver/deployment/inter-cluster-routing.md", + "sectionAnchor": "inter-cluster-routing", + "domain": "loadweaver" + } + }, + { + "id": "concept:loadweaver-topology", + "type": "concept", + "attrs": { + "title": "Topology", + "docPath": "docs/loadweaver/deployment/inter-cluster-routing.md", + "sectionAnchor": "topology", + "domain": "loadweaver" + } + }, + { + "id": "concept:loadweaver-why-bgp-not-ospf", + "type": "concept", + "attrs": { + "title": "Why BGP (not OSPF)", + "docPath": "docs/loadweaver/deployment/inter-cluster-routing.md", + "sectionAnchor": "why-bgp-not-ospf", + "domain": "loadweaver" + } + }, + { + "id": "concept:loadweaver-configuration-checklist", + "type": "concept", + "attrs": { + "title": "Configuration checklist", + "docPath": "docs/loadweaver/deployment/inter-cluster-routing.md", + "sectionAnchor": "configuration-checklist", + "domain": "loadweaver" + } + }, + { + "id": "concept:loadweaver-converge-order", + "type": "concept", + "attrs": { + "title": "Converge order", + "docPath": "docs/loadweaver/deployment/inter-cluster-routing.md", + "sectionAnchor": "converge-order", + "domain": "loadweaver" + } + }, + { + "id": "concept:loadweaver-safety-notes", + "type": "concept", + "attrs": { + "title": "Safety notes", + "docPath": "docs/loadweaver/deployment/inter-cluster-routing.md", + "sectionAnchor": "safety-notes", + "domain": "loadweaver" + } + }, + { + "id": "concept:loadweaver-lab-guide-3-vms", + "type": "concept", + "attrs": { + "title": "Lab guide (3 VMs)", + "docPath": "docs/loadweaver/deployment/lab-guide.md", + "sectionAnchor": "lab-guide-3-vms", + "domain": "loadweaver" + } + }, + { + "id": "concept:loadweaver-vm-sizing", + "type": "concept", + "attrs": { + "title": "VM sizing", + "docPath": "docs/loadweaver/deployment/lab-guide.md", + "sectionAnchor": "vm-sizing", + "domain": "loadweaver" + } + }, + { + "id": "concept:loadweaver-network", + "type": "concept", + "attrs": { + "title": "Network", + "docPath": "docs/loadweaver/deployment/lab-guide.md", + "sectionAnchor": "network", + "domain": "loadweaver" + } + }, + { + "id": "concept:loadweaver-operator-setup", + "type": "concept", + "attrs": { + "title": "Operator setup", + "docPath": "docs/loadweaver/deployment/lab-guide.md", + "sectionAnchor": "operator-setup", + "domain": "loadweaver" + } + }, + { + "id": "concept:loadweaver-converge-sequence", + "type": "concept", + "attrs": { + "title": "Converge sequence", + "docPath": "docs/loadweaver/deployment/lab-guide.md", + "sectionAnchor": "converge-sequence", + "domain": "loadweaver" + } + }, + { + "id": "concept:loadweaver-ssh-to-primary-edit-etc-loadweaver-traefik-acme-env-chmod-600-uncomment-cf-dns-api-token", + "type": "concept", + "attrs": { + "title": "SSH to primary: edit /etc/loadweaver/traefik-acme.env (chmod 600), uncomment CF_DNS_API_TOKEN=", + "docPath": "docs/loadweaver/deployment/lab-guide.md", + "sectionAnchor": "ssh-to-primary-edit-etc-loadweaver-traefik-acme-env-chmod-600-uncomment-cf-dns-api-token", + "domain": "loadweaver" + } + }, + { + "id": "concept:loadweaver-add-a-4th-node-incremental-update", + "type": "concept", + "attrs": { + "title": "Add a 4th node (incremental update)", + "docPath": "docs/loadweaver/deployment/lab-guide.md", + "sectionAnchor": "add-a-4th-node-incremental-update", + "domain": "loadweaver" + } + }, + { + "id": "concept:loadweaver-remove-a-node", + "type": "concept", + "attrs": { + "title": "Remove a node", + "docPath": "docs/loadweaver/deployment/lab-guide.md", + "sectionAnchor": "remove-a-node", + "domain": "loadweaver" + } + }, + { + "id": "concept:loadweaver-troubleshooting", + "type": "concept", + "attrs": { + "title": "Troubleshooting", + "docPath": "docs/loadweaver/deployment/lab-guide.md", + "sectionAnchor": "troubleshooting", + "domain": "loadweaver" + } + }, + { + "id": "concept:loadweaver-tear-down", + "type": "concept", + "attrs": { + "title": "Tear down", + "docPath": "docs/loadweaver/deployment/lab-guide.md", + "sectionAnchor": "tear-down", + "domain": "loadweaver" + } + }, + { + "id": "concept:loadweaver-production-tls-with-traefik-acme", + "type": "concept", + "attrs": { + "title": "Production TLS with Traefik ACME", + "docPath": "docs/loadweaver/deployment/production-tls.md", + "sectionAnchor": "production-tls-with-traefik-acme", + "domain": "loadweaver" + } + }, + { + "id": "concept:loadweaver-choose-a-challenge-type", + "type": "concept", + "attrs": { + "title": "Choose a challenge type", + "docPath": "docs/loadweaver/deployment/production-tls.md", + "sectionAnchor": "choose-a-challenge-type", + "domain": "loadweaver" + } + }, + { + "id": "concept:loadweaver-dns-01-recommended", + "type": "concept", + "attrs": { + "title": "DNS-01 (recommended)", + "docPath": "docs/loadweaver/deployment/production-tls.md", + "sectionAnchor": "dns-01-recommended", + "domain": "loadweaver" + } + }, + { + "id": "concept:loadweaver-cf-dns-api-token-your-scoped-token", + "type": "concept", + "attrs": { + "title": "CF_DNS_API_TOKEN=your-scoped-token", + "docPath": "docs/loadweaver/deployment/production-tls.md", + "sectionAnchor": "cf-dns-api-token-your-scoped-token", + "domain": "loadweaver" + } + }, + { + "id": "concept:loadweaver-http-01-only-when-port-80-is-public", + "type": "concept", + "attrs": { + "title": "HTTP-01 (only when port 80 is public)", + "docPath": "docs/loadweaver/deployment/production-tls.md", + "sectionAnchor": "http-01-only-when-port-80-is-public", + "domain": "loadweaver" + } + }, + { + "id": "concept:loadweaver-or-your-public-hostname-resolving-to-the-vip", + "type": "concept", + "attrs": { + "title": "or your public hostname resolving to the VIP", + "docPath": "docs/loadweaver/deployment/production-tls.md", + "sectionAnchor": "or-your-public-hostname-resolving-to-the-vip", + "domain": "loadweaver" + } + }, + { + "id": "concept:loadweaver-related-commands", + "type": "concept", + "attrs": { + "title": "Related commands", + "docPath": "docs/loadweaver/deployment/production-tls.md", + "sectionAnchor": "related-commands", + "domain": "loadweaver" + } + }, + { + "id": "concept:loadweaver-system-requirements", + "type": "concept", + "attrs": { + "title": "System requirements", + "docPath": "docs/loadweaver/deployment/system-requirements.md", + "sectionAnchor": "system-requirements", + "domain": "loadweaver" + } + }, + { + "id": "concept:loadweaver-operator-workstation", + "type": "concept", + "attrs": { + "title": "Operator workstation", + "docPath": "docs/loadweaver/deployment/system-requirements.md", + "sectionAnchor": "operator-workstation", + "domain": "loadweaver" + } + }, + { + "id": "concept:loadweaver-target-nodes", + "type": "concept", + "attrs": { + "title": "Target nodes", + "docPath": "docs/loadweaver/deployment/system-requirements.md", + "sectionAnchor": "target-nodes", + "domain": "loadweaver" + } + }, + { + "id": "concept:loadweaver-installed-automatically-by-host-bootstrap", + "type": "concept", + "attrs": { + "title": "Installed automatically by `host bootstrap`", + "docPath": "docs/loadweaver/deployment/system-requirements.md", + "sectionAnchor": "installed-automatically-by-host-bootstrap", + "domain": "loadweaver" + } + }, + { + "id": "concept:loadweaver-ports", + "type": "concept", + "attrs": { + "title": "Ports", + "docPath": "docs/loadweaver/deployment/system-requirements.md", + "sectionAnchor": "ports", + "domain": "loadweaver" + } + }, + { + "id": "concept:loadweaver-getting-started", + "type": "concept", + "attrs": { + "title": "Getting started", + "docPath": "docs/loadweaver/getting-started.md", + "sectionAnchor": "getting-started", + "domain": "loadweaver" + } + }, + { + "id": "concept:loadweaver-prerequisites", + "type": "concept", + "attrs": { + "title": "Prerequisites", + "docPath": "docs/loadweaver/getting-started.md", + "sectionAnchor": "prerequisites", + "domain": "loadweaver" + } + }, + { + "id": "concept:loadweaver-build-the-cli", + "type": "concept", + "attrs": { + "title": "Build the CLI", + "docPath": "docs/loadweaver/getting-started.md", + "sectionAnchor": "build-the-cli", + "domain": "loadweaver" + } + }, + { + "id": "concept:loadweaver-bootstrap-workflow", + "type": "concept", + "attrs": { + "title": "Bootstrap workflow", + "docPath": "docs/loadweaver/getting-started.md", + "sectionAnchor": "bootstrap-workflow", + "domain": "loadweaver" + } + }, + { + "id": "concept:loadweaver-typical-dependency-order", + "type": "concept", + "attrs": { + "title": "Typical dependency order", + "docPath": "docs/loadweaver/getting-started.md", + "sectionAnchor": "typical-dependency-order", + "domain": "loadweaver" + } + }, + { + "id": "concept:loadweaver-workspace-state", + "type": "concept", + "attrs": { + "title": "Workspace state", + "docPath": "docs/loadweaver/workspace.md", + "sectionAnchor": "workspace-state", + "domain": "loadweaver" + } + }, + { + "id": "concept:loadweaver-state-json", + "type": "concept", + "attrs": { + "title": "state.json", + "docPath": "docs/loadweaver/workspace.md", + "sectionAnchor": "state-json", + "domain": "loadweaver" + } + }, + { + "id": "concept:loadweaver-lock-json", + "type": "concept", + "attrs": { + "title": "lock.json", + "docPath": "docs/loadweaver/workspace.md", + "sectionAnchor": "lock-json", + "domain": "loadweaver" + } + }, + { + "id": "concept:loadweaver-host-inventory-and-host-locks", + "type": "concept", + "attrs": { + "title": "Host inventory and host locks", + "docPath": "docs/loadweaver/workspace.md", + "sectionAnchor": "host-inventory-and-host-locks", + "domain": "loadweaver" + } + }, + { + "id": "concept:loadweaver-wireguard-key-rotation-schedule", + "type": "concept", + "attrs": { + "title": "WireGuard key rotation schedule", + "docPath": "docs/loadweaver/workspace.md", + "sectionAnchor": "wireguard-key-rotation-schedule", + "domain": "loadweaver" + } + }, + { + "id": "concept:loadweaver-exit-1-keys-due-exit-2-warning-window", + "type": "concept", + "attrs": { + "title": "exit 1 = keys due, exit 2 = warning window", + "docPath": "docs/loadweaver/workspace.md", + "sectionAnchor": "exit-1-keys-due-exit-2-warning-window", + "domain": "loadweaver" + } + }, + { + "id": "concept:loadweaver-wireguard-keys-json", + "type": "concept", + "attrs": { + "title": "wireguard/keys.json", + "docPath": "docs/loadweaver/workspace.md", + "sectionAnchor": "wireguard-keys-json", + "domain": "loadweaver" + } + }, + { + "id": "concept:loadweaver-drift-detection", + "type": "concept", + "attrs": { + "title": "Drift detection", + "docPath": "docs/loadweaver/workspace.md", + "sectionAnchor": "drift-detection", + "domain": "loadweaver" + } + }, + { + "id": "concept:loadweaver-yes-semantics", + "type": "concept", + "attrs": { + "title": "--yes semantics", + "docPath": "docs/loadweaver/workspace.md", + "sectionAnchor": "yes-semantics", + "domain": "loadweaver" + } + }, { "id": "domain:shared", "type": "domain", @@ -29907,10 +31633,10 @@ } }, { - "id": "domain:identity", + "id": "domain:loadweaver", "type": "domain", "attrs": { - "name": "identity", + "name": "loadweaver", "kind": "domain", "label": "domain", "source": "tag" @@ -29926,6 +31652,16 @@ "source": "tag" } }, + { + "id": "domain:identity", + "type": "domain", + "attrs": { + "name": "identity", + "kind": "domain", + "label": "domain", + "source": "tag" + } + }, { "id": "feature-group:app", "type": "feature-group", @@ -30178,6 +31914,11 @@ "to": "project:shared-frontend-data-access-notifications", "type": "depends_on" }, + { + "from": "project:loadweaver-shared-feature-cli-wireguard", + "to": "project:loadweaver-shared-util-cli-core", + "type": "depends_on" + }, { "from": "project:agenstra-backend-feature-agent-manager", "to": "project:shared-backend-util-dynamic-provider-registry", @@ -30248,6 +31989,66 @@ "to": "project:shared-frontend-feature-landingpage", "type": "depends_on" }, + { + "from": "project:loadweaver-shared-feature-cli-cluster", + "to": "project:loadweaver-shared-util-cli-core", + "type": "depends_on" + }, + { + "from": "project:loadweaver-shared-feature-cli-cluster", + "to": "project:loadweaver-shared-feature-cli-ceph", + "type": "depends_on" + }, + { + "from": "project:loadweaver-shared-feature-cli-cluster", + "to": "project:loadweaver-shared-feature-cli-host", + "type": "depends_on" + }, + { + "from": "project:loadweaver-shared-feature-cli-cluster", + "to": "project:loadweaver-shared-feature-cli-node", + "type": "depends_on" + }, + { + "from": "project:loadweaver-shared-feature-cli-cluster", + "to": "project:loadweaver-shared-feature-cli-swarm", + "type": "depends_on" + }, + { + "from": "project:loadweaver-shared-feature-cli-cluster", + "to": "project:loadweaver-shared-feature-cli-traefik", + "type": "depends_on" + }, + { + "from": "project:loadweaver-shared-feature-cli-cluster", + "to": "project:loadweaver-shared-feature-cli-vip", + "type": "depends_on" + }, + { + "from": "project:loadweaver-shared-feature-cli-cluster", + "to": "project:loadweaver-shared-feature-cli-routing", + "type": "depends_on" + }, + { + "from": "project:loadweaver-shared-feature-cli-cluster", + "to": "project:loadweaver-shared-feature-cli-volume", + "type": "depends_on" + }, + { + "from": "project:loadweaver-shared-feature-cli-cluster", + "to": "project:loadweaver-shared-feature-cli-wireguard", + "type": "depends_on" + }, + { + "from": "project:loadweaver-shared-feature-cli-routing", + "to": "project:loadweaver-shared-util-cli-core", + "type": "depends_on" + }, + { + "from": "project:loadweaver-shared-feature-cli-traefik", + "to": "project:loadweaver-shared-util-cli-core", + "type": "depends_on" + }, { "from": "project:shared-frontend-feature-notifications", "to": "project:shared-frontend-data-access-notifications", @@ -30268,6 +32069,11 @@ "to": "project:shared-frontend-util-configuration", "type": "depends_on" }, + { + "from": "project:loadweaver-shared-feature-cli-volume", + "to": "project:loadweaver-shared-util-cli-core", + "type": "depends_on" + }, { "from": "project:shared-backend-feature-notifications", "to": "project:shared-backend-util-email", @@ -30293,6 +32099,11 @@ "to": "project:shared-backend-util-queue", "type": "depends_on" }, + { + "from": "project:loadweaver-shared-feature-cli-swarm", + "to": "project:loadweaver-shared-util-cli-core", + "type": "depends_on" + }, { "from": "project:shared-frontend-feature-landingpage", "to": "project:shared-frontend-data-access-communication", @@ -30318,16 +32129,131 @@ "to": "project:shared-frontend-util-runtime-config-server", "type": "depends_on" }, + { + "from": "project:loadweaver-shared-feature-cli-ceph", + "to": "project:loadweaver-shared-util-cli-core", + "type": "depends_on" + }, + { + "from": "project:loadweaver-shared-feature-cli-diag", + "to": "project:loadweaver-shared-util-cli-core", + "type": "depends_on" + }, + { + "from": "project:loadweaver-shared-feature-cli-diag", + "to": "project:loadweaver-shared-feature-cli-host", + "type": "depends_on" + }, + { + "from": "project:loadweaver-shared-feature-cli-diag", + "to": "project:loadweaver-shared-feature-cli-ceph", + "type": "depends_on" + }, + { + "from": "project:loadweaver-shared-feature-cli-diag", + "to": "project:loadweaver-shared-feature-cli-swarm", + "type": "depends_on" + }, + { + "from": "project:loadweaver-shared-feature-cli-diag", + "to": "project:loadweaver-shared-feature-cli-traefik", + "type": "depends_on" + }, + { + "from": "project:loadweaver-shared-feature-cli-diag", + "to": "project:loadweaver-shared-feature-cli-vip", + "type": "depends_on" + }, + { + "from": "project:loadweaver-shared-feature-cli-diag", + "to": "project:loadweaver-shared-feature-cli-volume", + "type": "depends_on" + }, + { + "from": "project:loadweaver-shared-feature-cli-diag", + "to": "project:loadweaver-shared-feature-cli-wireguard", + "type": "depends_on" + }, + { + "from": "project:loadweaver-shared-feature-cli-host", + "to": "project:loadweaver-shared-util-cli-core", + "type": "depends_on" + }, + { + "from": "project:loadweaver-shared-feature-cli-node", + "to": "project:loadweaver-shared-util-cli-core", + "type": "depends_on" + }, + { + "from": "project:loadweaver-shared-feature-cli-node", + "to": "project:loadweaver-shared-feature-cli-host", + "type": "depends_on" + }, + { + "from": "project:loadweaver-shared-feature-cli-node", + "to": "project:loadweaver-shared-feature-cli-ceph", + "type": "depends_on" + }, + { + "from": "project:loadweaver-shared-feature-cli-node", + "to": "project:loadweaver-shared-feature-cli-swarm", + "type": "depends_on" + }, + { + "from": "project:loadweaver-shared-feature-cli-node", + "to": "project:loadweaver-shared-feature-cli-volume", + "type": "depends_on" + }, + { + "from": "project:loadweaver-shared-feature-cli-node", + "to": "project:loadweaver-shared-feature-cli-wireguard", + "type": "depends_on" + }, + { + "from": "project:loadweaver-shared-feature-cli-ssh", + "to": "project:loadweaver-shared-util-cli-core", + "type": "depends_on" + }, + { + "from": "project:loadweaver-shared-feature-cli-vip", + "to": "project:loadweaver-shared-util-cli-core", + "type": "depends_on" + }, { "from": "project:shared-backend-feature-monitoring", "to": "project:identity-backend-util-auth", "type": "depends_on" }, + { + "from": "project:shared-shared-util-shell-executor", + "to": "project:shared-shared-util-logger", + "type": "depends_on" + }, { "from": "project:shared-backend-util-http-context", "to": "project:shared-backend-util-otel", "type": "depends_on" }, + { + "from": "project:loadweaver-shared-util-cli-core", + "to": "project:shared-shared-util-config-loader", + "type": "depends_on" + }, + { + "from": "project:loadweaver-shared-util-cli-core", + "to": "project:shared-shared-util-logger", + "type": "depends_on" + }, + { + "from": "project:loadweaver-shared-util-cli-core", + "to": "project:shared-shared-util-shell-executor", + "type": "depends_on" + }, + { + "from": "project:loadweaver-shared-util-cli-core", + "to": "project:shared-shared-util-ssh", + "type": "depends_on" + }, { "from": "project:shared-backend-util-redis-cache", "to": "project:shared-backend-util-otel", @@ -30373,6 +32299,91 @@ "to": "project:identity-backend-util-auth", "type": "depends_on" }, + { + "from": "project:loadweaver-shared-feature-cli", + "to": "project:shared-shared-util-config-loader", + "type": "depends_on" + }, + { + "from": "project:loadweaver-shared-feature-cli", + "to": "project:loadweaver-shared-util-cli-core", + "type": "depends_on" + }, + { + "from": "project:loadweaver-shared-feature-cli", + "to": "project:shared-shared-util-logger", + "type": "depends_on" + }, + { + "from": "project:loadweaver-shared-feature-cli", + "to": "project:shared-shared-util-shell-executor", + "type": "depends_on" + }, + { + "from": "project:loadweaver-shared-feature-cli", + "to": "project:shared-shared-util-ssh", + "type": "depends_on" + }, + { + "from": "project:loadweaver-shared-feature-cli", + "to": "project:loadweaver-shared-feature-cli-host", + "type": "depends_on" + }, + { + "from": "project:loadweaver-shared-feature-cli", + "to": "project:loadweaver-shared-feature-cli-cluster", + "type": "depends_on" + }, + { + "from": "project:loadweaver-shared-feature-cli", + "to": "project:loadweaver-shared-feature-cli-ceph", + "type": "depends_on" + }, + { + "from": "project:loadweaver-shared-feature-cli", + "to": "project:loadweaver-shared-feature-cli-diag", + "type": "depends_on" + }, + { + "from": "project:loadweaver-shared-feature-cli", + "to": "project:loadweaver-shared-feature-cli-node", + "type": "depends_on" + }, + { + "from": "project:loadweaver-shared-feature-cli", + "to": "project:loadweaver-shared-feature-cli-swarm", + "type": "depends_on" + }, + { + "from": "project:loadweaver-shared-feature-cli", + "to": "project:loadweaver-shared-feature-cli-traefik", + "type": "depends_on" + }, + { + "from": "project:loadweaver-shared-feature-cli", + "to": "project:loadweaver-shared-feature-cli-vip", + "type": "depends_on" + }, + { + "from": "project:loadweaver-shared-feature-cli", + "to": "project:loadweaver-shared-feature-cli-ssh", + "type": "depends_on" + }, + { + "from": "project:loadweaver-shared-feature-cli", + "to": "project:loadweaver-shared-feature-cli-routing", + "type": "depends_on" + }, + { + "from": "project:loadweaver-shared-feature-cli", + "to": "project:loadweaver-shared-feature-cli-volume", + "type": "depends_on" + }, + { + "from": "project:loadweaver-shared-feature-cli", + "to": "project:loadweaver-shared-feature-cli-wireguard", + "type": "depends_on" + }, { "from": "project:shared-frontend-feature-docs", "to": "project:shared-frontend-util-docs-parser", @@ -30653,6 +32664,11 @@ "to": "project:shared-backend-util-otel", "type": "depends_on" }, + { + "from": "project:shared-shared-util-ssh", + "to": "project:shared-shared-util-shell-executor", + "type": "depends_on" + }, { "from": "project:agenstra-frontend-landingpage", "to": "project:shared-frontend-util-cookie-consent", @@ -30728,6 +32744,11 @@ "to": "project:shared-frontend-util-express-server", "type": "depends_on" }, + { + "from": "project:loadweaver-cli-loadweaver", + "to": "project:loadweaver-shared-feature-cli", + "type": "depends_on" + }, { "from": "project:agenstra-frontend-docs", "to": "project:shared-frontend-docs", @@ -32513,6 +34534,21 @@ "to": "package:zone.js", "type": "depends_on" }, + { + "from": "project:loadweaver-cli-loadweaver", + "to": "package:commander", + "type": "depends_on" + }, + { + "from": "project:loadweaver-cli-loadweaver", + "to": "package:yaml", + "type": "depends_on" + }, + { + "from": "project:loadweaver-cli-loadweaver", + "to": "package:zod", + "type": "depends_on" + }, { "from": "project:agenstra-frontend-docs", "to": "package:@angular/animations", @@ -32763,6 +34799,311 @@ "to": "package:zod", "type": "depends_on" }, + { + "from": "project:forepath-frontend-data-access-project-estimator", + "to": "tool:jest", + "type": "depends_on" + }, + { + "from": "project:forepath-backend-feature-communication-manager", + "to": "tool:jest", + "type": "depends_on" + }, + { + "from": "project:decabill-frontend-data-access-billing-console", + "to": "tool:jest", + "type": "depends_on" + }, + { + "from": "project:shared-backend-util-dynamic-provider-registry", + "to": "tool:jest", + "type": "depends_on" + }, + { + "from": "project:agenstra-frontend-data-access-agent-console", + "to": "tool:jest", + "type": "depends_on" + }, + { + "from": "project:shared-frontend-util-provisioning-geography", + "to": "tool:jest", + "type": "depends_on" + }, + { + "from": "project:shared-backend-util-provisioning-geography", + "to": "tool:jest", + "type": "depends_on" + }, + { + "from": "project:shared-frontend-util-runtime-config-server", + "to": "tool:jest", + "type": "depends_on" + }, + { + "from": "project:agenstra-backend-feature-agent-controller", + "to": "tool:jest", + "type": "depends_on" + }, + { + "from": "project:decabill-frontend-feature-billing-console", + "to": "tool:jest", + "type": "depends_on" + }, + { + "from": "project:shared-frontend-data-access-communication", + "to": "tool:jest", + "type": "depends_on" + }, + { + "from": "project:shared-frontend-data-access-notifications", + "to": "tool:jest", + "type": "depends_on" + }, + { + "from": "project:decabill-backend-feature-billing-manager", + "to": "tool:jest", + "type": "depends_on" + }, + { + "from": "project:agenstra-frontend-feature-agent-console", + "to": "tool:jest", + "type": "depends_on" + }, + { + "from": "project:loadweaver-shared-feature-cli-wireguard", + "to": "tool:jest", + "type": "depends_on" + }, + { + "from": "project:agenstra-backend-feature-agent-manager", + "to": "tool:jest", + "type": "depends_on" + }, + { + "from": "project:shared-frontend-util-loading-indicator", + "to": "tool:jest", + "type": "depends_on" + }, + { + "from": "project:agenstra-frontend-feature-landingpage", + "to": "tool:jest", + "type": "depends_on" + }, + { + "from": "project:decabill-frontend-feature-landingpage", + "to": "tool:jest", + "type": "depends_on" + }, + { + "from": "project:forepath-frontend-feature-landingpage", + "to": "tool:jest", + "type": "depends_on" + }, + { + "from": "project:loadweaver-shared-feature-cli-cluster", + "to": "tool:jest", + "type": "depends_on" + }, + { + "from": "project:loadweaver-shared-feature-cli-routing", + "to": "tool:jest", + "type": "depends_on" + }, + { + "from": "project:loadweaver-shared-feature-cli-traefik", + "to": "tool:jest", + "type": "depends_on" + }, + { + "from": "project:shared-frontend-feature-notifications", + "to": "tool:jest", + "type": "depends_on" + }, + { + "from": "project:agenstra-frontend-data-access-portal", + "to": "tool:jest", + "type": "depends_on" + }, + { + "from": "project:decabill-frontend-data-access-portal", + "to": "tool:jest", + "type": "depends_on" + }, + { + "from": "project:loadweaver-shared-feature-cli-volume", + "to": "tool:jest", + "type": "depends_on" + }, + { + "from": "project:shared-backend-feature-notifications", + "to": "tool:jest", + "type": "depends_on" + }, + { + "from": "project:loadweaver-shared-feature-cli-swarm", + "to": "tool:jest", + "type": "depends_on" + }, + { + "from": "project:shared-backend-util-network-address", + "to": "tool:jest", + "type": "depends_on" + }, + { + "from": "project:shared-frontend-data-access-updates", + "to": "tool:jest", + "type": "depends_on" + }, + { + "from": "project:shared-frontend-feature-landingpage", + "to": "tool:jest", + "type": "depends_on" + }, + { + "from": "project:shared-frontend-util-cookie-consent", + "to": "tool:jest", + "type": "depends_on" + }, + { + "from": "project:shared-frontend-util-express-server", + "to": "tool:jest", + "type": "depends_on" + }, + { + "from": "project:identity-frontend-data-access-auth", + "to": "tool:jest", + "type": "depends_on" + }, + { + "from": "project:loadweaver-shared-feature-cli-ceph", + "to": "tool:jest", + "type": "depends_on" + }, + { + "from": "project:loadweaver-shared-feature-cli-diag", + "to": "tool:jest", + "type": "depends_on" + }, + { + "from": "project:loadweaver-shared-feature-cli-host", + "to": "tool:jest", + "type": "depends_on" + }, + { + "from": "project:loadweaver-shared-feature-cli-node", + "to": "tool:jest", + "type": "depends_on" + }, + { + "from": "project:shared-frontend-util-configuration", + "to": "tool:jest", + "type": "depends_on" + }, + { + "from": "project:shared-shared-util-network-address", + "to": "tool:jest", + "type": "depends_on" + }, + { + "from": "project:loadweaver-shared-feature-cli-ssh", + "to": "tool:jest", + "type": "depends_on" + }, + { + "from": "project:loadweaver-shared-feature-cli-vip", + "to": "tool:jest", + "type": "depends_on" + }, + { + "from": "project:shared-backend-feature-monitoring", + "to": "tool:jest", + "type": "depends_on" + }, + { + "from": "project:shared-shared-util-shell-executor", + "to": "tool:jest", + "type": "depends_on" + }, + { + "from": "project:shared-backend-util-http-context", + "to": "tool:jest", + "type": "depends_on" + }, + { + "from": "project:shared-frontend-util-docs-parser", + "to": "tool:jest", + "type": "depends_on" + }, + { + "from": "project:shared-shared-util-config-loader", + "to": "tool:jest", + "type": "depends_on" + }, + { + "from": "project:loadweaver-shared-util-cli-core", + "to": "tool:jest", + "type": "depends_on" + }, + { + "from": "project:shared-backend-util-redis-cache", + "to": "tool:jest", + "type": "depends_on" + }, + { + "from": "project:shared-frontend-feature-updates", + "to": "tool:jest", + "type": "depends_on" + }, + { + "from": "project:identity-frontend-feature-auth", + "to": "tool:jest", + "type": "depends_on" + }, + { + "from": "project:shared-backend-feature-updates", + "to": "tool:jest", + "type": "depends_on" + }, + { + "from": "project:shared-backend-util-opensearch", + "to": "tool:jest", + "type": "depends_on" + }, + { + "from": "project:identity-backend-feature-auth", + "to": "tool:jest", + "type": "depends_on" + }, + { + "from": "project:loadweaver-shared-feature-cli", + "to": "tool:jest", + "type": "depends_on" + }, + { + "from": "project:shared-frontend-feature-docs", + "to": "tool:jest", + "type": "depends_on" + }, + { + "from": "project:identity-frontend-util-auth", + "to": "tool:jest", + "type": "depends_on" + }, + { + "from": "project:shared-backend-util-webhook", + "to": "tool:jest", + "type": "depends_on" + }, + { + "from": "project:identity-backend-util-auth", + "to": "tool:jest", + "type": "depends_on" + }, + { + "from": "project:shared-backend-util-crypto", + "to": "tool:jest", + "type": "depends_on" + }, { "from": "project:agenstra-backend-agent-controller", "to": "project:sbom", @@ -32773,6 +35114,11 @@ "to": "tool:ci", "type": "depends_on" }, + { + "from": "project:agenstra-backend-agent-controller", + "to": "tool:jest", + "type": "depends_on" + }, { "from": "project:agenstra-frontend-billing-console", "to": "project:sbom", @@ -32783,6 +35129,11 @@ "to": "tool:ci", "type": "depends_on" }, + { + "from": "project:agenstra-frontend-billing-console", + "to": "tool:jest", + "type": "depends_on" + }, { "from": "project:decabill-frontend-billing-console", "to": "project:sbom", @@ -32793,6 +35144,11 @@ "to": "tool:ci", "type": "depends_on" }, + { + "from": "project:decabill-frontend-billing-console", + "to": "tool:jest", + "type": "depends_on" + }, { "from": "project:forepath-frontend-billing-console", "to": "project:sbom", @@ -32803,6 +35159,31 @@ "to": "tool:ci", "type": "depends_on" }, + { + "from": "project:forepath-frontend-billing-console", + "to": "tool:jest", + "type": "depends_on" + }, + { + "from": "project:shared-backend-util-email", + "to": "tool:jest", + "type": "depends_on" + }, + { + "from": "project:shared-backend-util-queue", + "to": "tool:jest", + "type": "depends_on" + }, + { + "from": "project:shared-frontend-util-meta", + "to": "tool:jest", + "type": "depends_on" + }, + { + "from": "project:shared-shared-util-logger", + "to": "tool:jest", + "type": "depends_on" + }, { "from": "project:decabill-backend-billing-manager", "to": "project:sbom", @@ -32813,6 +35194,21 @@ "to": "tool:ci", "type": "depends_on" }, + { + "from": "project:decabill-backend-billing-manager", + "to": "tool:jest", + "type": "depends_on" + }, + { + "from": "project:shared-backend-util-otel", + "to": "tool:jest", + "type": "depends_on" + }, + { + "from": "project:shared-frontend-ui-lists", + "to": "tool:jest", + "type": "depends_on" + }, { "from": "project:agenstra-frontend-agent-console", "to": "project:sbom", @@ -32823,6 +35219,11 @@ "to": "tool:ci", "type": "depends_on" }, + { + "from": "project:agenstra-frontend-agent-console", + "to": "tool:jest", + "type": "depends_on" + }, { "from": "project:agenstra-backend-agent-manager", "to": "project:sbom", @@ -32833,16 +35234,36 @@ "to": "tool:ci", "type": "depends_on" }, + { + "from": "project:agenstra-backend-agent-manager", + "to": "tool:jest", + "type": "depends_on" + }, { "from": "project:forepath-backend-communication", "to": "project:sbom", "type": "depends_on" }, + { + "from": "project:forepath-backend-communication", + "to": "tool:jest", + "type": "depends_on" + }, + { + "from": "project:shared-shared-util-ssh", + "to": "tool:jest", + "type": "depends_on" + }, { "from": "project:agenstra-frontend-landingpage", "to": "project:sbom", "type": "depends_on" }, + { + "from": "project:agenstra-frontend-landingpage", + "to": "tool:jest", + "type": "depends_on" + }, { "from": "project:agenstra-native-agent-console", "to": "project:sbom", @@ -32853,6 +35274,21 @@ "to": "project:sbom", "type": "depends_on" }, + { + "from": "project:decabill-frontend-landingpage", + "to": "tool:jest", + "type": "depends_on" + }, + { + "from": "project:forepath-frontend-landingpage", + "to": "tool:jest", + "type": "depends_on" + }, + { + "from": "project:loadweaver-cli-loadweaver", + "to": "tool:jest", + "type": "depends_on" + }, { "from": "project:agenstra-frontend-docs", "to": "project:sbom", @@ -32863,6 +35299,11 @@ "to": "tool:docs", "type": "depends_on" }, + { + "from": "project:agenstra-frontend-docs", + "to": "tool:jest", + "type": "depends_on" + }, { "from": "project:decabill-frontend-docs", "to": "project:sbom", @@ -32873,16 +35314,36 @@ "to": "tool:docs", "type": "depends_on" }, + { + "from": "project:decabill-frontend-docs", + "to": "tool:jest", + "type": "depends_on" + }, + { + "from": "project:shared-frontend-docs", + "to": "tool:jest", + "type": "depends_on" + }, { "from": "project:shared-mcp-devkit", "to": "project:sbom", "type": "depends_on" }, + { + "from": "project:shared-mcp-devkit", + "to": "tool:jest", + "type": "depends_on" + }, { "from": "project:shared-mcp-proxy", "to": "project:sbom", "type": "depends_on" }, + { + "from": "project:shared-mcp-proxy", + "to": "tool:jest", + "type": "depends_on" + }, { "from": "project:forepath-frontend-data-access-project-estimator", "to": "file:libs/domains/forepath/frontend/data-access-project-estimator/README.md", @@ -36218,6 +38679,16 @@ "to": "file:libs/domains/agenstra/frontend/feature-agent-console/src/lib/theme.service.ts", "type": "contains" }, + { + "from": "project:loadweaver-shared-feature-cli-wireguard", + "to": "file:libs/domains/loadweaver/shared/feature-cli-wireguard/README.md", + "type": "contains" + }, + { + "from": "project:loadweaver-shared-feature-cli-wireguard", + "to": "file:libs/domains/loadweaver/shared/feature-cli-wireguard/src/lib/wireguard.service.ts", + "type": "contains" + }, { "from": "project:agenstra-backend-feature-agent-manager", "to": "file:libs/domains/agenstra/backend/feature-agent-manager/README.md", @@ -36848,6 +39319,41 @@ "to": "file:libs/domains/forepath/frontend/feature-landingpage/README.md", "type": "contains" }, + { + "from": "project:loadweaver-shared-feature-cli-cluster", + "to": "file:libs/domains/loadweaver/shared/feature-cli-cluster/README.md", + "type": "contains" + }, + { + "from": "project:loadweaver-shared-feature-cli-cluster", + "to": "file:libs/domains/loadweaver/shared/feature-cli-cluster/src/lib/node-inventory.service.ts", + "type": "contains" + }, + { + "from": "project:loadweaver-shared-feature-cli-routing", + "to": "file:libs/domains/loadweaver/shared/feature-cli-routing/src/lib/routing-cross-wg.service.ts", + "type": "contains" + }, + { + "from": "project:loadweaver-shared-feature-cli-routing", + "to": "file:libs/domains/loadweaver/shared/feature-cli-routing/src/lib/routing.service.ts", + "type": "contains" + }, + { + "from": "project:loadweaver-shared-feature-cli-traefik", + "to": "file:libs/domains/loadweaver/shared/feature-cli-traefik/README.md", + "type": "contains" + }, + { + "from": "project:loadweaver-shared-feature-cli-traefik", + "to": "file:libs/domains/loadweaver/shared/feature-cli-traefik/src/lib/traefik-acme-env.service.ts", + "type": "contains" + }, + { + "from": "project:loadweaver-shared-feature-cli-traefik", + "to": "file:libs/domains/loadweaver/shared/feature-cli-traefik/src/lib/traefik.service.ts", + "type": "contains" + }, { "from": "project:shared-frontend-feature-notifications", "to": "file:libs/domains/shared/frontend/feature-notifications/README.md", @@ -36883,6 +39389,16 @@ "to": "file:libs/domains/decabill/frontend/data-access-portal/src/lib/state/service-plans", "type": "contains" }, + { + "from": "project:loadweaver-shared-feature-cli-volume", + "to": "file:libs/domains/loadweaver/shared/feature-cli-volume/README.md", + "type": "contains" + }, + { + "from": "project:loadweaver-shared-feature-cli-volume", + "to": "file:libs/domains/loadweaver/shared/feature-cli-volume/src/lib/volume.service.ts", + "type": "contains" + }, { "from": "project:shared-backend-feature-notifications", "to": "file:libs/domains/shared/backend/feature-notifications/README.md", @@ -36963,6 +39479,16 @@ "to": "file:libs/domains/shared/backend/feature-notifications/src/lib/services/webhook-endpoint.service.ts", "type": "contains" }, + { + "from": "project:loadweaver-shared-feature-cli-swarm", + "to": "file:libs/domains/loadweaver/shared/feature-cli-swarm/README.md", + "type": "contains" + }, + { + "from": "project:loadweaver-shared-feature-cli-swarm", + "to": "file:libs/domains/loadweaver/shared/feature-cli-swarm/src/lib/swarm.service.ts", + "type": "contains" + }, { "from": "project:shared-backend-util-network-address", "to": "file:libs/domains/shared/backend/util-network-address/README.md", @@ -37013,6 +39539,41 @@ "to": "file:libs/domains/identity/frontend/data-access-auth/src/lib/state/personal-access-tokens", "type": "contains" }, + { + "from": "project:loadweaver-shared-feature-cli-ceph", + "to": "file:libs/domains/loadweaver/shared/feature-cli-ceph/README.md", + "type": "contains" + }, + { + "from": "project:loadweaver-shared-feature-cli-ceph", + "to": "file:libs/domains/loadweaver/shared/feature-cli-ceph/src/lib/ceph.service.ts", + "type": "contains" + }, + { + "from": "project:loadweaver-shared-feature-cli-diag", + "to": "file:libs/domains/loadweaver/shared/feature-cli-diag/README.md", + "type": "contains" + }, + { + "from": "project:loadweaver-shared-feature-cli-diag", + "to": "file:libs/domains/loadweaver/shared/feature-cli-diag/src/lib/diag.service.ts", + "type": "contains" + }, + { + "from": "project:loadweaver-shared-feature-cli-host", + "to": "file:libs/domains/loadweaver/shared/feature-cli-host/src/lib/host.service.ts", + "type": "contains" + }, + { + "from": "project:loadweaver-shared-feature-cli-node", + "to": "file:libs/domains/loadweaver/shared/feature-cli-node/README.md", + "type": "contains" + }, + { + "from": "project:loadweaver-shared-feature-cli-node", + "to": "file:libs/domains/loadweaver/shared/feature-cli-node/src/lib/node.service.ts", + "type": "contains" + }, { "from": "project:shared-frontend-util-configuration", "to": "file:libs/domains/shared/frontend/util-configuration/README.md", @@ -37028,6 +39589,41 @@ "to": "file:libs/domains/shared/shared/util-network-address/README.md", "type": "contains" }, + { + "from": "project:loadweaver-shared-feature-cli-ssh", + "to": "file:libs/domains/loadweaver/shared/feature-cli-ssh/README.md", + "type": "contains" + }, + { + "from": "project:loadweaver-shared-feature-cli-ssh", + "to": "file:libs/domains/loadweaver/shared/feature-cli-ssh/src/lib/join-remote-command.service.ts", + "type": "contains" + }, + { + "from": "project:loadweaver-shared-feature-cli-ssh", + "to": "file:libs/domains/loadweaver/shared/feature-cli-ssh/src/lib/ssh.service.ts", + "type": "contains" + }, + { + "from": "project:loadweaver-shared-feature-cli-vip", + "to": "file:libs/domains/loadweaver/shared/feature-cli-vip/README.md", + "type": "contains" + }, + { + "from": "project:loadweaver-shared-feature-cli-vip", + "to": "file:libs/domains/loadweaver/shared/feature-cli-vip/src/lib/backend-resolution.service.ts", + "type": "contains" + }, + { + "from": "project:loadweaver-shared-feature-cli-vip", + "to": "file:libs/domains/loadweaver/shared/feature-cli-vip/src/lib/vip-inspection.service.ts", + "type": "contains" + }, + { + "from": "project:loadweaver-shared-feature-cli-vip", + "to": "file:libs/domains/loadweaver/shared/feature-cli-vip/src/lib/vip.service.ts", + "type": "contains" + }, { "from": "project:shared-backend-feature-monitoring", "to": "file:libs/domains/shared/backend/feature-monitoring/README.md", @@ -37043,6 +39639,11 @@ "to": "file:libs/domains/shared/backend/feature-monitoring/src/lib/monitoring.module.ts", "type": "contains" }, + { + "from": "project:shared-shared-util-shell-executor", + "to": "file:libs/domains/shared/shared/util-shell-executor/README.md", + "type": "contains" + }, { "from": "project:shared-backend-util-http-context", "to": "file:libs/domains/shared/backend/util-http-context/README.md", @@ -37073,6 +39674,71 @@ "to": "file:libs/domains/shared/frontend/util-docs-parser/src/lib/services/search-index-builder.service.ts", "type": "contains" }, + { + "from": "project:shared-shared-util-config-loader", + "to": "file:libs/domains/shared/shared/util-config-loader/README.md", + "type": "contains" + }, + { + "from": "project:loadweaver-shared-util-cli-core", + "to": "file:libs/domains/loadweaver/shared/util-cli-core/README.md", + "type": "contains" + }, + { + "from": "project:loadweaver-shared-util-cli-core", + "to": "file:libs/domains/loadweaver/shared/util-cli-core/src/lib/services/derive-cluster-cidr.service.ts", + "type": "contains" + }, + { + "from": "project:loadweaver-shared-util-cli-core", + "to": "file:libs/domains/loadweaver/shared/util-cli-core/src/lib/services/derive-osd-devices.service.ts", + "type": "contains" + }, + { + "from": "project:loadweaver-shared-util-cli-core", + "to": "file:libs/domains/loadweaver/shared/util-cli-core/src/lib/services/derive-vip-state.service.ts", + "type": "contains" + }, + { + "from": "project:loadweaver-shared-util-cli-core", + "to": "file:libs/domains/loadweaver/shared/util-cli-core/src/lib/services/expected-swarm-labels.service.ts", + "type": "contains" + }, + { + "from": "project:loadweaver-shared-util-cli-core", + "to": "file:libs/domains/loadweaver/shared/util-cli-core/src/lib/services/node-registry.service.ts", + "type": "contains" + }, + { + "from": "project:loadweaver-shared-util-cli-core", + "to": "file:libs/domains/loadweaver/shared/util-cli-core/src/lib/services/prerequisite.service.ts", + "type": "contains" + }, + { + "from": "project:loadweaver-shared-util-cli-core", + "to": "file:libs/domains/loadweaver/shared/util-cli-core/src/lib/services/remote-exec.service.ts", + "type": "contains" + }, + { + "from": "project:loadweaver-shared-util-cli-core", + "to": "file:libs/domains/loadweaver/shared/util-cli-core/src/lib/services/resolve-routing-hubs.service.ts", + "type": "contains" + }, + { + "from": "project:loadweaver-shared-util-cli-core", + "to": "file:libs/domains/loadweaver/shared/util-cli-core/src/lib/services/routing-state.service.ts", + "type": "contains" + }, + { + "from": "project:loadweaver-shared-util-cli-core", + "to": "file:libs/domains/loadweaver/shared/util-cli-core/src/lib/services/ssh-config-validation.service.ts", + "type": "contains" + }, + { + "from": "project:loadweaver-shared-util-cli-core", + "to": "file:libs/domains/loadweaver/shared/util-cli-core/src/lib/services/ssh-target.service.ts", + "type": "contains" + }, { "from": "project:shared-backend-util-redis-cache", "to": "file:libs/domains/shared/backend/util-redis-cache/README.md", @@ -37363,6 +40029,56 @@ "to": "file:libs/domains/identity/backend/feature-auth/src/lib/services/users.service.ts", "type": "contains" }, + { + "from": "project:loadweaver-shared-feature-cli", + "to": "file:libs/domains/loadweaver/shared/feature-cli/README.md", + "type": "contains" + }, + { + "from": "project:loadweaver-shared-feature-cli", + "to": "file:libs/domains/loadweaver/shared/feature-cli/docs/architecture.mmd", + "type": "contains" + }, + { + "from": "project:loadweaver-shared-feature-cli", + "to": "file:libs/domains/loadweaver/shared/feature-cli/docs/cli-overview.mmd", + "type": "contains" + }, + { + "from": "project:loadweaver-shared-feature-cli", + "to": "file:libs/domains/loadweaver/shared/feature-cli/docs/cluster-init-sequence.mmd", + "type": "contains" + }, + { + "from": "project:loadweaver-shared-feature-cli", + "to": "file:libs/domains/loadweaver/shared/feature-cli/docs/cluster-topology.mmd", + "type": "contains" + }, + { + "from": "project:loadweaver-shared-feature-cli", + "to": "file:libs/domains/loadweaver/shared/feature-cli/docs/cluster-update-planner.mmd", + "type": "contains" + }, + { + "from": "project:loadweaver-shared-feature-cli", + "to": "file:libs/domains/loadweaver/shared/feature-cli/docs/network-layers.mmd", + "type": "contains" + }, + { + "from": "project:loadweaver-shared-feature-cli", + "to": "file:libs/domains/loadweaver/shared/feature-cli/docs/routing-architecture.mmd", + "type": "contains" + }, + { + "from": "project:loadweaver-shared-feature-cli", + "to": "file:libs/domains/loadweaver/shared/feature-cli/docs/routing-inter-cluster.mmd", + "type": "contains" + }, + { + "from": "project:loadweaver-shared-feature-cli", + "to": "file:libs/domains/loadweaver/shared/feature-cli/docs/workspace-state.mmd", + "type": "contains" + }, { "from": "project:shared-frontend-feature-docs", "to": "file:libs/domains/shared/frontend/feature-docs/README.md", @@ -37523,6 +40239,11 @@ "to": "file:libs/domains/shared/frontend/util-meta/README.md", "type": "contains" }, + { + "from": "project:shared-shared-util-logger", + "to": "file:libs/domains/shared/shared/util-logger/README.md", + "type": "contains" + }, { "from": "project:decabill-backend-billing-manager", "to": "file:apps/decabill/backend-billing-manager/src/app/app.module.ts", @@ -37583,6 +40304,11 @@ "to": "file:apps/shared/platform-authentication/README.md", "type": "contains" }, + { + "from": "project:shared-shared-util-ssh", + "to": "file:libs/domains/shared/shared/util-ssh/README.md", + "type": "contains" + }, { "from": "project:agenstra-frontend-landingpage", "to": "file:apps/agenstra/frontend-landingpage/src/app/viewport-scroller-offset.service.ts", @@ -48118,6 +50844,386 @@ "to": "concept:forepath-api-reference", "type": "contains" }, + { + "from": "file:docs/loadweaver/README.md", + "to": "concept:loadweaver-loadweaver", + "type": "contains" + }, + { + "from": "file:docs/loadweaver/README.md", + "to": "concept:loadweaver-quick-start", + "type": "contains" + }, + { + "from": "file:docs/loadweaver/README.md", + "to": "concept:loadweaver-documentation", + "type": "contains" + }, + { + "from": "file:docs/loadweaver/architecture/system-overview.md", + "to": "concept:loadweaver-architecture-overview", + "type": "contains" + }, + { + "from": "file:docs/loadweaver/architecture/system-overview.md", + "to": "concept:loadweaver-execution-model", + "type": "contains" + }, + { + "from": "file:docs/loadweaver/architecture/system-overview.md", + "to": "concept:loadweaver-future-extensions", + "type": "contains" + }, + { + "from": "file:docs/loadweaver/cli-reference.md", + "to": "concept:loadweaver-cli-reference", + "type": "contains" + }, + { + "from": "file:docs/loadweaver/cli-reference.md", + "to": "concept:loadweaver-commands", + "type": "contains" + }, + { + "from": "file:docs/loadweaver/cli-reference.md", + "to": "concept:loadweaver-progress-output", + "type": "contains" + }, + { + "from": "file:docs/loadweaver/cli-reference.md", + "to": "concept:loadweaver-mutation-safety", + "type": "contains" + }, + { + "from": "file:docs/loadweaver/cli-reference.md", + "to": "concept:loadweaver-wireguard-key-rotation", + "type": "contains" + }, + { + "from": "file:docs/loadweaver/cli-reference.md", + "to": "concept:loadweaver-rotate-all-node-keys-and-roll-out-updated-peer-configs", + "type": "contains" + }, + { + "from": "file:docs/loadweaver/cli-reference.md", + "to": "concept:loadweaver-rotate-a-single-node", + "type": "contains" + }, + { + "from": "file:docs/loadweaver/cli-reference.md", + "to": "concept:loadweaver-inspect-key-age-and-due-warning-state", + "type": "contains" + }, + { + "from": "file:docs/loadweaver/cli-reference.md", + "to": "concept:loadweaver-exit-codes-for-monitoring-when-wireguard-keyrotation-enabled-is-true", + "type": "contains" + }, + { + "from": "file:docs/loadweaver/cli-reference.md", + "to": "concept:loadweaver-0-all-keys-ok-1-at-least-one-key-due-2-warning-window-only", + "type": "contains" + }, + { + "from": "file:docs/loadweaver/cli-reference.md", + "to": "concept:loadweaver-rotate-only-overdue-keys-for-cron-systemd-automation", + "type": "contains" + }, + { + "from": "file:docs/loadweaver/cli-reference.md", + "to": "concept:loadweaver-print-suggested-cron-systemd-timer-snippets", + "type": "contains" + }, + { + "from": "file:docs/loadweaver/cli-reference.md", + "to": "concept:loadweaver-swarm-label-reconcile", + "type": "contains" + }, + { + "from": "file:docs/loadweaver/cli-reference.md", + "to": "concept:loadweaver-ad-hoc-ssh", + "type": "contains" + }, + { + "from": "file:docs/loadweaver/configuration.md", + "to": "concept:loadweaver-configuration", + "type": "contains" + }, + { + "from": "file:docs/loadweaver/configuration.md", + "to": "concept:loadweaver-secrets-and-config-overlays", + "type": "contains" + }, + { + "from": "file:docs/loadweaver/configuration.md", + "to": "concept:loadweaver-only-override-keepalived-password-keep-the-rest-from-loadweaver-yml", + "type": "contains" + }, + { + "from": "file:docs/loadweaver/configuration.md", + "to": "concept:loadweaver-key-sections", + "type": "contains" + }, + { + "from": "file:docs/loadweaver/configuration.md", + "to": "concept:loadweaver-ssh-access", + "type": "contains" + }, + { + "from": "file:docs/loadweaver/configuration.md", + "to": "concept:loadweaver-advanced-routing-bird", + "type": "contains" + }, + { + "from": "file:docs/loadweaver/configuration.md", + "to": "concept:loadweaver-host-bootstrap", + "type": "contains" + }, + { + "from": "file:docs/loadweaver/configuration.md", + "to": "concept:loadweaver-ceph-osd-scale-out", + "type": "contains" + }, + { + "from": "file:docs/loadweaver/configuration.md", + "to": "concept:loadweaver-traefik-acme-production-tls", + "type": "contains" + }, + { + "from": "file:docs/loadweaver/configuration.md", + "to": "concept:loadweaver-on-primary-edit-etc-loadweaver-traefik-acme-env-chmod-600", + "type": "contains" + }, + { + "from": "file:docs/loadweaver/configuration.md", + "to": "concept:loadweaver-vip-failover-verification", + "type": "contains" + }, + { + "from": "file:docs/loadweaver/deployment/inter-cluster-routing.md", + "to": "concept:loadweaver-inter-cluster-routing", + "type": "contains" + }, + { + "from": "file:docs/loadweaver/deployment/inter-cluster-routing.md", + "to": "concept:loadweaver-topology", + "type": "contains" + }, + { + "from": "file:docs/loadweaver/deployment/inter-cluster-routing.md", + "to": "concept:loadweaver-why-bgp-not-ospf", + "type": "contains" + }, + { + "from": "file:docs/loadweaver/deployment/inter-cluster-routing.md", + "to": "concept:loadweaver-configuration-checklist", + "type": "contains" + }, + { + "from": "file:docs/loadweaver/deployment/inter-cluster-routing.md", + "to": "concept:loadweaver-converge-order", + "type": "contains" + }, + { + "from": "file:docs/loadweaver/deployment/inter-cluster-routing.md", + "to": "concept:loadweaver-safety-notes", + "type": "contains" + }, + { + "from": "file:docs/loadweaver/deployment/lab-guide.md", + "to": "concept:loadweaver-lab-guide-3-vms", + "type": "contains" + }, + { + "from": "file:docs/loadweaver/deployment/lab-guide.md", + "to": "concept:loadweaver-vm-sizing", + "type": "contains" + }, + { + "from": "file:docs/loadweaver/deployment/lab-guide.md", + "to": "concept:loadweaver-network", + "type": "contains" + }, + { + "from": "file:docs/loadweaver/deployment/lab-guide.md", + "to": "concept:loadweaver-operator-setup", + "type": "contains" + }, + { + "from": "file:docs/loadweaver/deployment/lab-guide.md", + "to": "concept:loadweaver-configuration", + "type": "contains" + }, + { + "from": "file:docs/loadweaver/deployment/lab-guide.md", + "to": "concept:loadweaver-converge-sequence", + "type": "contains" + }, + { + "from": "file:docs/loadweaver/deployment/lab-guide.md", + "to": "concept:loadweaver-ssh-to-primary-edit-etc-loadweaver-traefik-acme-env-chmod-600-uncomment-cf-dns-api-token", + "type": "contains" + }, + { + "from": "file:docs/loadweaver/deployment/lab-guide.md", + "to": "concept:loadweaver-add-a-4th-node-incremental-update", + "type": "contains" + }, + { + "from": "file:docs/loadweaver/deployment/lab-guide.md", + "to": "concept:loadweaver-remove-a-node", + "type": "contains" + }, + { + "from": "file:docs/loadweaver/deployment/lab-guide.md", + "to": "concept:loadweaver-troubleshooting", + "type": "contains" + }, + { + "from": "file:docs/loadweaver/deployment/lab-guide.md", + "to": "concept:loadweaver-tear-down", + "type": "contains" + }, + { + "from": "file:docs/loadweaver/deployment/production-tls.md", + "to": "concept:loadweaver-production-tls-with-traefik-acme", + "type": "contains" + }, + { + "from": "file:docs/loadweaver/deployment/production-tls.md", + "to": "concept:loadweaver-choose-a-challenge-type", + "type": "contains" + }, + { + "from": "file:docs/loadweaver/deployment/production-tls.md", + "to": "concept:loadweaver-dns-01-recommended", + "type": "contains" + }, + { + "from": "file:docs/loadweaver/deployment/production-tls.md", + "to": "concept:loadweaver-cf-dns-api-token-your-scoped-token", + "type": "contains" + }, + { + "from": "file:docs/loadweaver/deployment/production-tls.md", + "to": "concept:loadweaver-http-01-only-when-port-80-is-public", + "type": "contains" + }, + { + "from": "file:docs/loadweaver/deployment/production-tls.md", + "to": "concept:loadweaver-or-your-public-hostname-resolving-to-the-vip", + "type": "contains" + }, + { + "from": "file:docs/loadweaver/deployment/production-tls.md", + "to": "concept:loadweaver-troubleshooting", + "type": "contains" + }, + { + "from": "file:docs/loadweaver/deployment/production-tls.md", + "to": "concept:loadweaver-related-commands", + "type": "contains" + }, + { + "from": "file:docs/loadweaver/deployment/system-requirements.md", + "to": "concept:loadweaver-system-requirements", + "type": "contains" + }, + { + "from": "file:docs/loadweaver/deployment/system-requirements.md", + "to": "concept:loadweaver-operator-workstation", + "type": "contains" + }, + { + "from": "file:docs/loadweaver/deployment/system-requirements.md", + "to": "concept:loadweaver-target-nodes", + "type": "contains" + }, + { + "from": "file:docs/loadweaver/deployment/system-requirements.md", + "to": "concept:loadweaver-installed-automatically-by-host-bootstrap", + "type": "contains" + }, + { + "from": "file:docs/loadweaver/deployment/system-requirements.md", + "to": "concept:loadweaver-network", + "type": "contains" + }, + { + "from": "file:docs/loadweaver/deployment/system-requirements.md", + "to": "concept:loadweaver-ports", + "type": "contains" + }, + { + "from": "file:docs/loadweaver/getting-started.md", + "to": "concept:loadweaver-getting-started", + "type": "contains" + }, + { + "from": "file:docs/loadweaver/getting-started.md", + "to": "concept:loadweaver-prerequisites", + "type": "contains" + }, + { + "from": "file:docs/loadweaver/getting-started.md", + "to": "concept:loadweaver-build-the-cli", + "type": "contains" + }, + { + "from": "file:docs/loadweaver/getting-started.md", + "to": "concept:loadweaver-bootstrap-workflow", + "type": "contains" + }, + { + "from": "file:docs/loadweaver/getting-started.md", + "to": "concept:loadweaver-typical-dependency-order", + "type": "contains" + }, + { + "from": "file:docs/loadweaver/workspace.md", + "to": "concept:loadweaver-workspace-state", + "type": "contains" + }, + { + "from": "file:docs/loadweaver/workspace.md", + "to": "concept:loadweaver-state-json", + "type": "contains" + }, + { + "from": "file:docs/loadweaver/workspace.md", + "to": "concept:loadweaver-lock-json", + "type": "contains" + }, + { + "from": "file:docs/loadweaver/workspace.md", + "to": "concept:loadweaver-host-inventory-and-host-locks", + "type": "contains" + }, + { + "from": "file:docs/loadweaver/workspace.md", + "to": "concept:loadweaver-wireguard-key-rotation-schedule", + "type": "contains" + }, + { + "from": "file:docs/loadweaver/workspace.md", + "to": "concept:loadweaver-exit-1-keys-due-exit-2-warning-window", + "type": "contains" + }, + { + "from": "file:docs/loadweaver/workspace.md", + "to": "concept:loadweaver-wireguard-keys-json", + "type": "contains" + }, + { + "from": "file:docs/loadweaver/workspace.md", + "to": "concept:loadweaver-drift-detection", + "type": "contains" + }, + { + "from": "file:docs/loadweaver/workspace.md", + "to": "concept:loadweaver-yes-semantics", + "type": "contains" + }, { "from": "file:libs/domains/forepath/backend/feature-communication-manager/src/lib/controllers/public-contact-requests.controller.ts", "to": "api:HTTP:POST:/public/contact-requests", @@ -68588,6 +71694,96 @@ "to": "api:HTTP:GET:/otel/metrics", "type": "documents" }, + { + "from": "concept:loadweaver-quick-start", + "to": "project:loadweaver-cli-loadweaver", + "type": "documents" + }, + { + "from": "concept:loadweaver-documentation", + "to": "api:HTTP:GET:/config", + "type": "documents" + }, + { + "from": "concept:loadweaver-future-extensions", + "to": "api:HTTP:GET:/config", + "type": "documents" + }, + { + "from": "concept:loadweaver-configuration", + "to": "api:HTTP:GET:/config", + "type": "documents" + }, + { + "from": "concept:loadweaver-secrets-and-config-overlays", + "to": "api:HTTP:GET:/config", + "type": "documents" + }, + { + "from": "concept:loadweaver-only-override-keepalived-password-keep-the-rest-from-loadweaver-yml", + "to": "api:HTTP:GET:/config", + "type": "documents" + }, + { + "from": "concept:loadweaver-key-sections", + "to": "api:HTTP:GET:/config", + "type": "documents" + }, + { + "from": "concept:loadweaver-ssh-access", + "to": "api:HTTP:GET:/config", + "type": "documents" + }, + { + "from": "concept:loadweaver-advanced-routing-bird", + "to": "api:HTTP:GET:/config", + "type": "documents" + }, + { + "from": "concept:loadweaver-host-bootstrap", + "to": "api:HTTP:GET:/config", + "type": "documents" + }, + { + "from": "concept:loadweaver-ceph-osd-scale-out", + "to": "api:HTTP:POST:/auth/register", + "type": "documents" + }, + { + "from": "concept:loadweaver-ceph-osd-scale-out", + "to": "api:HTTP:GET:/config", + "type": "documents" + }, + { + "from": "concept:loadweaver-traefik-acme-production-tls", + "to": "api:HTTP:GET:/config", + "type": "documents" + }, + { + "from": "concept:loadweaver-on-primary-edit-etc-loadweaver-traefik-acme-env-chmod-600", + "to": "api:HTTP:GET:/config", + "type": "documents" + }, + { + "from": "concept:loadweaver-vip-failover-verification", + "to": "api:HTTP:GET:/config", + "type": "documents" + }, + { + "from": "concept:loadweaver-operator-setup", + "to": "project:loadweaver-cli-loadweaver", + "type": "documents" + }, + { + "from": "concept:loadweaver-prerequisites", + "to": "api:HTTP:GET:/config", + "type": "documents" + }, + { + "from": "concept:loadweaver-build-the-cli", + "to": "project:loadweaver-cli-loadweaver", + "type": "documents" + }, { "from": "project:@forepath/test/mounted-plugin-fixture", "to": "domain:shared", @@ -68808,6 +72004,21 @@ "to": "feature-group:feature", "type": "belongs_to" }, + { + "from": "project:loadweaver-shared-feature-cli-wireguard", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "project:loadweaver-shared-feature-cli-wireguard", + "to": "context:shared", + "type": "belongs_to" + }, + { + "from": "project:loadweaver-shared-feature-cli-wireguard", + "to": "feature-group:feature", + "type": "belongs_to" + }, { "from": "project:agenstra-backend-feature-agent-manager", "to": "domain:agenstra", @@ -68883,6 +72094,51 @@ "to": "feature-group:feature", "type": "belongs_to" }, + { + "from": "project:loadweaver-shared-feature-cli-cluster", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "project:loadweaver-shared-feature-cli-cluster", + "to": "context:shared", + "type": "belongs_to" + }, + { + "from": "project:loadweaver-shared-feature-cli-cluster", + "to": "feature-group:feature", + "type": "belongs_to" + }, + { + "from": "project:loadweaver-shared-feature-cli-routing", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "project:loadweaver-shared-feature-cli-routing", + "to": "context:shared", + "type": "belongs_to" + }, + { + "from": "project:loadweaver-shared-feature-cli-routing", + "to": "feature-group:feature", + "type": "belongs_to" + }, + { + "from": "project:loadweaver-shared-feature-cli-traefik", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "project:loadweaver-shared-feature-cli-traefik", + "to": "context:shared", + "type": "belongs_to" + }, + { + "from": "project:loadweaver-shared-feature-cli-traefik", + "to": "feature-group:feature", + "type": "belongs_to" + }, { "from": "project:shared-frontend-feature-notifications", "to": "domain:shared", @@ -68928,6 +72184,21 @@ "to": "feature-group:data-access", "type": "belongs_to" }, + { + "from": "project:loadweaver-shared-feature-cli-volume", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "project:loadweaver-shared-feature-cli-volume", + "to": "context:shared", + "type": "belongs_to" + }, + { + "from": "project:loadweaver-shared-feature-cli-volume", + "to": "feature-group:feature", + "type": "belongs_to" + }, { "from": "project:shared-backend-feature-notifications", "to": "domain:shared", @@ -68943,6 +72214,21 @@ "to": "feature-group:feature", "type": "belongs_to" }, + { + "from": "project:loadweaver-shared-feature-cli-swarm", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "project:loadweaver-shared-feature-cli-swarm", + "to": "context:shared", + "type": "belongs_to" + }, + { + "from": "project:loadweaver-shared-feature-cli-swarm", + "to": "feature-group:feature", + "type": "belongs_to" + }, { "from": "project:shared-backend-util-network-address", "to": "domain:shared", @@ -69033,6 +72319,66 @@ "to": "feature-group:data-access", "type": "belongs_to" }, + { + "from": "project:loadweaver-shared-feature-cli-ceph", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "project:loadweaver-shared-feature-cli-ceph", + "to": "context:shared", + "type": "belongs_to" + }, + { + "from": "project:loadweaver-shared-feature-cli-ceph", + "to": "feature-group:feature", + "type": "belongs_to" + }, + { + "from": "project:loadweaver-shared-feature-cli-diag", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "project:loadweaver-shared-feature-cli-diag", + "to": "context:shared", + "type": "belongs_to" + }, + { + "from": "project:loadweaver-shared-feature-cli-diag", + "to": "feature-group:feature", + "type": "belongs_to" + }, + { + "from": "project:loadweaver-shared-feature-cli-host", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "project:loadweaver-shared-feature-cli-host", + "to": "context:shared", + "type": "belongs_to" + }, + { + "from": "project:loadweaver-shared-feature-cli-host", + "to": "feature-group:feature", + "type": "belongs_to" + }, + { + "from": "project:loadweaver-shared-feature-cli-node", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "project:loadweaver-shared-feature-cli-node", + "to": "context:shared", + "type": "belongs_to" + }, + { + "from": "project:loadweaver-shared-feature-cli-node", + "to": "feature-group:feature", + "type": "belongs_to" + }, { "from": "project:shared-frontend-util-configuration", "to": "domain:shared", @@ -69063,6 +72409,36 @@ "to": "feature-group:util", "type": "belongs_to" }, + { + "from": "project:loadweaver-shared-feature-cli-ssh", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "project:loadweaver-shared-feature-cli-ssh", + "to": "context:shared", + "type": "belongs_to" + }, + { + "from": "project:loadweaver-shared-feature-cli-ssh", + "to": "feature-group:feature", + "type": "belongs_to" + }, + { + "from": "project:loadweaver-shared-feature-cli-vip", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "project:loadweaver-shared-feature-cli-vip", + "to": "context:shared", + "type": "belongs_to" + }, + { + "from": "project:loadweaver-shared-feature-cli-vip", + "to": "feature-group:feature", + "type": "belongs_to" + }, { "from": "project:shared-backend-feature-monitoring", "to": "domain:shared", @@ -69078,6 +72454,21 @@ "to": "feature-group:feature", "type": "belongs_to" }, + { + "from": "project:shared-shared-util-shell-executor", + "to": "domain:shared", + "type": "belongs_to" + }, + { + "from": "project:shared-shared-util-shell-executor", + "to": "context:shared", + "type": "belongs_to" + }, + { + "from": "project:shared-shared-util-shell-executor", + "to": "feature-group:util", + "type": "belongs_to" + }, { "from": "project:shared-backend-util-http-context", "to": "domain:shared", @@ -69108,6 +72499,36 @@ "to": "feature-group:util", "type": "belongs_to" }, + { + "from": "project:shared-shared-util-config-loader", + "to": "domain:shared", + "type": "belongs_to" + }, + { + "from": "project:shared-shared-util-config-loader", + "to": "context:shared", + "type": "belongs_to" + }, + { + "from": "project:shared-shared-util-config-loader", + "to": "feature-group:util", + "type": "belongs_to" + }, + { + "from": "project:loadweaver-shared-util-cli-core", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "project:loadweaver-shared-util-cli-core", + "to": "context:shared", + "type": "belongs_to" + }, + { + "from": "project:loadweaver-shared-util-cli-core", + "to": "feature-group:util", + "type": "belongs_to" + }, { "from": "project:shared-backend-util-redis-cache", "to": "domain:shared", @@ -69198,6 +72619,21 @@ "to": "feature-group:feature", "type": "belongs_to" }, + { + "from": "project:loadweaver-shared-feature-cli", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "project:loadweaver-shared-feature-cli", + "to": "context:shared", + "type": "belongs_to" + }, + { + "from": "project:loadweaver-shared-feature-cli", + "to": "feature-group:feature", + "type": "belongs_to" + }, { "from": "project:shared-frontend-feature-docs", "to": "domain:shared", @@ -69378,6 +72814,21 @@ "to": "feature-group:util", "type": "belongs_to" }, + { + "from": "project:shared-shared-util-logger", + "to": "domain:shared", + "type": "belongs_to" + }, + { + "from": "project:shared-shared-util-logger", + "to": "context:shared", + "type": "belongs_to" + }, + { + "from": "project:shared-shared-util-logger", + "to": "feature-group:util", + "type": "belongs_to" + }, { "from": "project:decabill-backend-billing-manager", "to": "domain:decabill", @@ -69483,6 +72934,21 @@ "to": "feature-group:app", "type": "belongs_to" }, + { + "from": "project:shared-shared-util-ssh", + "to": "domain:shared", + "type": "belongs_to" + }, + { + "from": "project:shared-shared-util-ssh", + "to": "context:shared", + "type": "belongs_to" + }, + { + "from": "project:shared-shared-util-ssh", + "to": "feature-group:util", + "type": "belongs_to" + }, { "from": "project:agenstra-frontend-landingpage", "to": "domain:agenstra", @@ -69543,6 +73009,21 @@ "to": "feature-group:app", "type": "belongs_to" }, + { + "from": "project:loadweaver-cli-loadweaver", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "project:loadweaver-cli-loadweaver", + "to": "context:shared", + "type": "belongs_to" + }, + { + "from": "project:loadweaver-cli-loadweaver", + "to": "feature-group:app", + "type": "belongs_to" + }, { "from": "project:agenstra-frontend-docs", "to": "domain:agenstra", @@ -70293,6 +73774,56 @@ "to": "domain:forepath", "type": "belongs_to" }, + { + "from": "file:docs/loadweaver/README.md", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "file:docs/loadweaver/architecture/system-overview.md", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "file:docs/loadweaver/cli-reference.md", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "file:docs/loadweaver/configuration.md", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "file:docs/loadweaver/deployment/inter-cluster-routing.md", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "file:docs/loadweaver/deployment/lab-guide.md", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "file:docs/loadweaver/deployment/production-tls.md", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "file:docs/loadweaver/deployment/system-requirements.md", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "file:docs/loadweaver/getting-started.md", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "file:docs/loadweaver/workspace.md", + "to": "domain:loadweaver", + "type": "belongs_to" + }, { "from": "concept:agenstra-agenstra-documentation", "to": "domain:agenstra", @@ -75287,6 +78818,371 @@ "from": "concept:forepath-api-reference", "to": "domain:forepath", "type": "belongs_to" + }, + { + "from": "concept:loadweaver-loadweaver", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "concept:loadweaver-quick-start", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "concept:loadweaver-documentation", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "concept:loadweaver-architecture-overview", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "concept:loadweaver-execution-model", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "concept:loadweaver-future-extensions", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "concept:loadweaver-cli-reference", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "concept:loadweaver-commands", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "concept:loadweaver-progress-output", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "concept:loadweaver-mutation-safety", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "concept:loadweaver-wireguard-key-rotation", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "concept:loadweaver-rotate-all-node-keys-and-roll-out-updated-peer-configs", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "concept:loadweaver-rotate-a-single-node", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "concept:loadweaver-inspect-key-age-and-due-warning-state", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "concept:loadweaver-exit-codes-for-monitoring-when-wireguard-keyrotation-enabled-is-true", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "concept:loadweaver-0-all-keys-ok-1-at-least-one-key-due-2-warning-window-only", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "concept:loadweaver-rotate-only-overdue-keys-for-cron-systemd-automation", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "concept:loadweaver-print-suggested-cron-systemd-timer-snippets", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "concept:loadweaver-swarm-label-reconcile", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "concept:loadweaver-ad-hoc-ssh", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "concept:loadweaver-configuration", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "concept:loadweaver-secrets-and-config-overlays", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "concept:loadweaver-only-override-keepalived-password-keep-the-rest-from-loadweaver-yml", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "concept:loadweaver-key-sections", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "concept:loadweaver-ssh-access", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "concept:loadweaver-advanced-routing-bird", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "concept:loadweaver-host-bootstrap", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "concept:loadweaver-ceph-osd-scale-out", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "concept:loadweaver-traefik-acme-production-tls", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "concept:loadweaver-on-primary-edit-etc-loadweaver-traefik-acme-env-chmod-600", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "concept:loadweaver-vip-failover-verification", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "concept:loadweaver-inter-cluster-routing", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "concept:loadweaver-topology", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "concept:loadweaver-why-bgp-not-ospf", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "concept:loadweaver-configuration-checklist", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "concept:loadweaver-converge-order", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "concept:loadweaver-safety-notes", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "concept:loadweaver-lab-guide-3-vms", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "concept:loadweaver-vm-sizing", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "concept:loadweaver-network", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "concept:loadweaver-operator-setup", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "concept:loadweaver-converge-sequence", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "concept:loadweaver-ssh-to-primary-edit-etc-loadweaver-traefik-acme-env-chmod-600-uncomment-cf-dns-api-token", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "concept:loadweaver-add-a-4th-node-incremental-update", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "concept:loadweaver-remove-a-node", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "concept:loadweaver-troubleshooting", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "concept:loadweaver-tear-down", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "concept:loadweaver-production-tls-with-traefik-acme", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "concept:loadweaver-choose-a-challenge-type", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "concept:loadweaver-dns-01-recommended", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "concept:loadweaver-cf-dns-api-token-your-scoped-token", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "concept:loadweaver-http-01-only-when-port-80-is-public", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "concept:loadweaver-or-your-public-hostname-resolving-to-the-vip", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "concept:loadweaver-related-commands", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "concept:loadweaver-system-requirements", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "concept:loadweaver-operator-workstation", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "concept:loadweaver-target-nodes", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "concept:loadweaver-installed-automatically-by-host-bootstrap", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "concept:loadweaver-ports", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "concept:loadweaver-getting-started", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "concept:loadweaver-prerequisites", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "concept:loadweaver-build-the-cli", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "concept:loadweaver-bootstrap-workflow", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "concept:loadweaver-typical-dependency-order", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "concept:loadweaver-workspace-state", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "concept:loadweaver-state-json", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "concept:loadweaver-lock-json", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "concept:loadweaver-host-inventory-and-host-locks", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "concept:loadweaver-wireguard-key-rotation-schedule", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "concept:loadweaver-exit-1-keys-due-exit-2-warning-window", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "concept:loadweaver-wireguard-keys-json", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "concept:loadweaver-drift-detection", + "to": "domain:loadweaver", + "type": "belongs_to" + }, + { + "from": "concept:loadweaver-yes-semantics", + "to": "domain:loadweaver", + "type": "belongs_to" } ] } diff --git a/libs/domains/loadweaver/keycloak/index.ts b/libs/domains/loadweaver/keycloak/index.ts new file mode 100644 index 000000000..b818789fe --- /dev/null +++ b/libs/domains/loadweaver/keycloak/index.ts @@ -0,0 +1,2 @@ +// loadweaver domain keycloak exports +export const NAME = 'loadweaver-keycloak'; diff --git a/libs/domains/loadweaver/shared/feature-cli-ceph/.eslintrc.json b/libs/domains/loadweaver/shared/feature-cli-ceph/.eslintrc.json new file mode 100644 index 000000000..b33995654 --- /dev/null +++ b/libs/domains/loadweaver/shared/feature-cli-ceph/.eslintrc.json @@ -0,0 +1,18 @@ +{ + "extends": ["../../../../../.eslintrc.json"], + "ignorePatterns": ["!**/*"], + "overrides": [ + { + "files": ["*.ts", "*.tsx", "*.js", "*.jsx"], + "rules": {} + }, + { + "files": ["*.ts", "*.tsx"], + "rules": {} + }, + { + "files": ["*.js", "*.jsx"], + "rules": {} + } + ] +} diff --git a/libs/domains/loadweaver/shared/feature-cli-ceph/README.md b/libs/domains/loadweaver/shared/feature-cli-ceph/README.md new file mode 100644 index 000000000..fbcabb875 --- /dev/null +++ b/libs/domains/loadweaver/shared/feature-cli-ceph/README.md @@ -0,0 +1,7 @@ +# loadweaver-shared-feature-cli-ceph + +This library was generated with [Nx](https://nx.dev). + +## Running unit tests + +Run `nx test loadweaver-shared-feature-cli-ceph` to execute the unit tests via [Jest](https://jestjs.io). diff --git a/libs/domains/loadweaver/shared/feature-cli-ceph/jest.config.cts b/libs/domains/loadweaver/shared/feature-cli-ceph/jest.config.cts new file mode 100644 index 000000000..9415fb9de --- /dev/null +++ b/libs/domains/loadweaver/shared/feature-cli-ceph/jest.config.cts @@ -0,0 +1,11 @@ +module.exports = { + displayName: 'loadweaver-shared-feature-cli-ceph', + preset: '../../../../../jest.preset.cjs', + testEnvironment: 'node', + transform: { + '^.+\\.[tj]s$': ['ts-jest', { tsconfig: '/tsconfig.spec.json' }], + }, + moduleFileExtensions: ['ts', 'js', 'html'], + coverageDirectory: + '../../../../../coverage/libs/domains/loadweaver/shared/feature-cli-ceph', +}; diff --git a/libs/domains/loadweaver/shared/feature-cli-ceph/project.json b/libs/domains/loadweaver/shared/feature-cli-ceph/project.json new file mode 100644 index 000000000..31bc57684 --- /dev/null +++ b/libs/domains/loadweaver/shared/feature-cli-ceph/project.json @@ -0,0 +1,16 @@ +{ + "name": "loadweaver-shared-feature-cli-ceph", + "$schema": "../../../../../node_modules/nx/schemas/project-schema.json", + "sourceRoot": "libs/domains/loadweaver/shared/feature-cli-ceph/src", + "projectType": "library", + "tags": ["domain:loadweaver", "scope:shared", "type:feature"], + "targets": { + "test": { + "executor": "@nx/jest:jest", + "outputs": ["{workspaceRoot}/coverage/{projectRoot}"], + "options": { + "jestConfig": "libs/domains/loadweaver/shared/feature-cli-ceph/jest.config.cts" + } + } + } +} diff --git a/libs/domains/loadweaver/shared/feature-cli-ceph/src/index.ts b/libs/domains/loadweaver/shared/feature-cli-ceph/src/index.ts new file mode 100644 index 000000000..f6ce23916 --- /dev/null +++ b/libs/domains/loadweaver/shared/feature-cli-ceph/src/index.ts @@ -0,0 +1,2 @@ +export { registerCephCommands } from './lib/command'; +export { CephService } from './lib/ceph.service'; diff --git a/libs/domains/loadweaver/shared/feature-cli-ceph/src/lib/ceph.service.ts b/libs/domains/loadweaver/shared/feature-cli-ceph/src/lib/ceph.service.ts new file mode 100644 index 000000000..dcff396d7 --- /dev/null +++ b/libs/domains/loadweaver/shared/feature-cli-ceph/src/lib/ceph.service.ts @@ -0,0 +1,235 @@ +import type { LoadweaverContext } from '@forepath/loadweaver/shared/util-cli-core'; +import { + assertRemoteSuccess, + deriveOsdDevices, + isRemoteAlreadyExists, + printStructuredOutput, +} from '@forepath/loadweaver/shared/util-cli-core'; + +export class CephService { + constructor(private readonly ctx: LoadweaverContext) {} + + async init(): Promise { + const config = this.requireConfig(); + const monNodes = Object.entries(config.nodes).filter(([, node]) => node.roles.includes('ceph-mon')); + + for (const [nodeId] of monNodes) { + const check = await this.ctx + .sshForNode(nodeId) + .execRemote('command -v cephadm', { dryRun: this.ctx.options.dryRun }); + + if (check.exitCode !== 0) { + throw new Error(`cephadm is not installed on ${nodeId}. Run loadweaver host bootstrap first.`); + } + } + + const primary = config.cluster.primaryManager; + const bootstrap = await this.ctx + .sshForNode(primary) + .execRemote("cephadm bootstrap --mon-ip $(hostname -I | awk '{print $1}')", { + dryRun: this.ctx.options.dryRun, + }); + + if (bootstrap.exitCode !== 0 && !isRemoteAlreadyExists(bootstrap)) { + const status = await this.ctx.sshForNode(primary).execRemote('ceph -s', { dryRun: this.ctx.options.dryRun }); + + if (status.exitCode !== 0) { + assertRemoteSuccess(bootstrap, 'Ceph bootstrap'); + } + } + + await this.reconcileOsds(); + } + + async createCephfs(): Promise { + const config = this.requireConfig(); + const primary = config.cluster.primaryManager; + const result = await this.ctx + .sshForNode(primary) + .execRemote(`ceph fs volume create ${config.ceph.fsName}`, { dryRun: this.ctx.options.dryRun }); + + if (result.exitCode !== 0 && !isRemoteAlreadyExists(result)) { + assertRemoteSuccess(result, `Create CephFS ${config.ceph.fsName}`); + } + } + + async mountCephfsAll(): Promise { + await this.mountCephfs(Object.keys(this.requireConfig().nodes)); + } + + async mountCephfs(nodeIds: string[]): Promise { + const config = this.requireConfig(); + + for (const nodeId of nodeIds) { + if (!config.nodes[nodeId]) { + continue; + } + + const mountPath = config.ceph.mountPath; + const result = await this.ctx + .sshForNode(nodeId) + .execRemote( + `mkdir -p ${mountPath} && if mountpoint -q ${mountPath}; then exit 0; else mount -t ceph :${config.ceph.fsName} ${mountPath}; fi`, + { dryRun: this.ctx.options.dryRun }, + ); + + assertRemoteSuccess(result, `Mount CephFS on ${nodeId}`); + } + } + + async reconcileOsds(): Promise { + const devices = deriveOsdDevices(this.requireConfig()); + + for (const nodeId of Object.keys(devices).sort()) { + await this.addOsdForNode(nodeId); + } + } + + async addOsdForNode(nodeId: string): Promise { + const config = this.requireConfig(); + const node = config.nodes[nodeId]; + + if (!node) { + throw new Error(`Unknown node: ${nodeId}`); + } + + const device = node.osdDevice; + + if (!device) { + throw new Error(`Node ${nodeId} has no osdDevice configured`); + } + + await this.addOsd(nodeId, device); + } + + async addOsd(nodeId: string, device: string): Promise { + const config = this.requireConfig(); + const node = config.nodes[nodeId]; + + if (!node) { + throw new Error(`Unknown node: ${nodeId}`); + } + + const primary = config.cluster.primaryManager; + const hostName = node.hostname; + const hostIp = node.wireguardIp; + + const hostAdd = await this.ctx.sshForNode(primary).execRemote(`ceph orch host add ${hostName} ${hostIp}`, { + dryRun: this.ctx.options.dryRun, + }); + + if (hostAdd.exitCode !== 0 && !isRemoteAlreadyExists(hostAdd)) { + assertRemoteSuccess(hostAdd, `Add Ceph host ${hostName}`); + } + + const result = await this.ctx + .sshForNode(primary) + .execRemote(`ceph orch daemon add osd ${hostName}:${device}`, { dryRun: this.ctx.options.dryRun }); + + if (result.exitCode !== 0 && !isRemoteAlreadyExists(result)) { + assertRemoteSuccess(result, `Add OSD on ${hostName}:${device}`); + } + } + + async removeOsdsForNode(nodeId: string, hostnameOverride?: string): Promise { + const config = this.requireConfig(); + const hostName = hostnameOverride ?? config.nodes[nodeId]?.hostname; + + if (!hostName) { + this.ctx.logger.warn(`Skipping OSD removal for ${nodeId}: hostname unknown`); + return; + } + + const primary = config.cluster.primaryManager; + const listResult = await this.ctx + .sshForNode(primary) + .execRemote(`ceph orch ps --hostname ${hostName} --daemon_type osd --format '{{.daemon_name}}'`, { + dryRun: this.ctx.options.dryRun, + }); + + assertRemoteSuccess(listResult, `List OSD daemons on ${hostName}`); + + const daemonNames = listResult.stdout + .split('\n') + .map((line) => line.trim()) + .filter(Boolean); + + for (const daemonName of daemonNames) { + const osdId = daemonName.startsWith('osd.') ? daemonName : `osd.${daemonName.replace(/^ceph-osd\./, '')}`; + const removeResult = await this.ctx + .sshForNode(primary) + .execRemote(`ceph osd out ${osdId} && ceph orch osd rm ${osdId} --zap`, { + dryRun: this.ctx.options.dryRun, + }); + + if (removeResult.exitCode !== 0 && !this.ctx.options.dryRun) { + assertRemoteSuccess(removeResult, `Remove OSD ${osdId} from ${hostName}`); + } + } + + const hostRemove = await this.ctx + .sshForNode(primary) + .execRemote(`ceph orch host rm ${hostName} --force`, { dryRun: this.ctx.options.dryRun }); + + if (hostRemove.exitCode !== 0 && !isRemoteAlreadyExists(hostRemove) && daemonNames.length > 0) { + assertRemoteSuccess(hostRemove, `Remove Ceph host ${hostName}`); + } + } + + async removeOsd(osdId: string): Promise { + const primary = this.requireConfig().cluster.primaryManager; + const normalized = osdId.startsWith('osd.') ? osdId : `osd.${osdId}`; + const result = await this.ctx + .sshForNode(primary) + .execRemote(`ceph osd out ${normalized} && ceph orch osd rm ${normalized} --zap`, { + dryRun: this.ctx.options.dryRun, + }); + + assertRemoteSuccess(result, `Remove OSD ${normalized}`); + } + + async status(): Promise { + const snapshot = await this.inspectStatus(); + + if (this.ctx.options.json) { + printStructuredOutput(this.ctx, snapshot); + return; + } + + console.log(snapshot.output || snapshot.stderr); + } + + async inspectStatus(): Promise<{ exitCode: number; output: string; stderr: string }> { + const primary = this.requireConfig().cluster.primaryManager; + const result = await this.ctx + .sshForNode(primary) + .execRemote('ceph -s && ceph fs status && ceph orch device ls', { dryRun: this.ctx.options.dryRun }); + + return { + exitCode: result.exitCode, + output: result.stdout, + stderr: result.stderr, + }; + } + + async unmount(nodeId: string): Promise { + const config = this.requireConfig(); + const mountPath = config.ceph.mountPath; + + await this.ctx.sshForNode(nodeId).execRemote(`umount ${mountPath} || true`, { dryRun: this.ctx.options.dryRun }); + } + + async unmountAll(): Promise { + for (const nodeId of Object.keys(this.requireConfig().nodes)) { + await this.unmount(nodeId); + } + } + + private requireConfig() { + if (!this.ctx.config) { + throw new Error('Configuration not loaded'); + } + + return this.ctx.config; + } +} diff --git a/libs/domains/loadweaver/shared/feature-cli-ceph/src/lib/command.ts b/libs/domains/loadweaver/shared/feature-cli-ceph/src/lib/command.ts new file mode 100644 index 000000000..bf37b7b09 --- /dev/null +++ b/libs/domains/loadweaver/shared/feature-cli-ceph/src/lib/command.ts @@ -0,0 +1,78 @@ +import type { Command } from 'commander'; + +import type { LoadweaverContext } from '@forepath/loadweaver/shared/util-cli-core'; +import { runGuardedMutation, withExamples } from '@forepath/loadweaver/shared/util-cli-core'; + +import { CephService } from './ceph.service'; + +export function registerCephCommands(program: Command, getCtx: (command: Command) => LoadweaverContext): void { + const ceph = program.command('ceph').description('Ceph cluster and CephFS management'); + withExamples(ceph, ['loadweaver ceph status', 'loadweaver --dry-run ceph init']); + + const init = ceph + .command('init') + .description('Bootstrap Ceph MON/MGR via cephadm') + .action(async function (this: Command) { + const ctx = getCtx(this); + await runGuardedMutation(ctx, 'ceph.init', () => new CephService(ctx).init()); + }); + withExamples(init, ['loadweaver ceph init']); + + const cephfsCreate = ceph + .command('cephfs-create') + .description('Create CephFS filesystem') + .action(async function (this: Command) { + const ctx = getCtx(this); + await runGuardedMutation(ctx, 'ceph.cephfs-create', () => new CephService(ctx).createCephfs()); + }); + withExamples(cephfsCreate, ['loadweaver ceph cephfs-create']); + + const cephfsMount = ceph + .command('cephfs-mount') + .description('Mount CephFS on all nodes') + .action(async function (this: Command) { + const ctx = getCtx(this); + await runGuardedMutation(ctx, 'ceph.cephfs-mount', () => new CephService(ctx).mountCephfsAll()); + }); + withExamples(cephfsMount, ['loadweaver ceph cephfs-mount']); + + const osdAdd = ceph + .command('osd-add') + .description('Add an OSD on a node') + .argument('', 'Node identifier') + .argument('[device]', 'Block device path (defaults to nodes..osdDevice)') + .action(async function (this: Command, nodeId: string, device?: string) { + const ctx = getCtx(this); + await runGuardedMutation(ctx, `ceph.osd-add.${nodeId}`, () => + device ? new CephService(ctx).addOsd(nodeId, device) : new CephService(ctx).addOsdForNode(nodeId), + ); + }); + withExamples(osdAdd, ['loadweaver ceph osd-add node-a1 /dev/sdb', 'loadweaver ceph osd-add node-a2']); + + const osdReconcile = ceph + .command('osd-reconcile') + .description('Add OSDs for all nodes with ceph-osd role and osdDevice configured') + .action(async function (this: Command) { + const ctx = getCtx(this); + await runGuardedMutation(ctx, 'ceph.osd-reconcile', () => new CephService(ctx).reconcileOsds()); + }); + withExamples(osdReconcile, ['loadweaver ceph osd-reconcile']); + + const osdRemove = ceph + .command('osd-remove') + .description('Remove an OSD') + .argument('', 'OSD identifier') + .action(async function (this: Command, osdId: string) { + const ctx = getCtx(this); + await runGuardedMutation(ctx, `ceph.osd-remove.${osdId}`, () => new CephService(ctx).removeOsd(osdId)); + }); + withExamples(osdRemove, ['loadweaver ceph osd-remove osd.0']); + + const status = ceph + .command('status') + .description('Show Ceph cluster health') + .action(async function (this: Command) { + await new CephService(getCtx(this)).status(); + }); + withExamples(status, ['loadweaver ceph status']); +} diff --git a/libs/domains/loadweaver/shared/feature-cli-ceph/tsconfig.json b/libs/domains/loadweaver/shared/feature-cli-ceph/tsconfig.json new file mode 100644 index 000000000..9e5fde1c6 --- /dev/null +++ b/libs/domains/loadweaver/shared/feature-cli-ceph/tsconfig.json @@ -0,0 +1,16 @@ +{ + "extends": "../../../../../tsconfig.base.json", + "compilerOptions": { + "module": "commonjs" + }, + "files": [], + "include": [], + "references": [ + { + "path": "./tsconfig.lib.json" + }, + { + "path": "./tsconfig.spec.json" + } + ] +} diff --git a/libs/domains/loadweaver/shared/feature-cli-ceph/tsconfig.lib.json b/libs/domains/loadweaver/shared/feature-cli-ceph/tsconfig.lib.json new file mode 100644 index 000000000..841e950a0 --- /dev/null +++ b/libs/domains/loadweaver/shared/feature-cli-ceph/tsconfig.lib.json @@ -0,0 +1,15 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "../../../../../dist/out-tsc", + "declaration": true, + "types": ["node"] + }, + "include": ["src/**/*.ts"], + "exclude": [ + "jest.config.ts", + "jest.config.cts", + "src/**/*.spec.ts", + "src/**/*.test.ts" + ] +} diff --git a/libs/domains/loadweaver/shared/feature-cli-ceph/tsconfig.spec.json b/libs/domains/loadweaver/shared/feature-cli-ceph/tsconfig.spec.json new file mode 100644 index 000000000..c714b3c64 --- /dev/null +++ b/libs/domains/loadweaver/shared/feature-cli-ceph/tsconfig.spec.json @@ -0,0 +1,16 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "../../../../../dist/out-tsc", + "module": "commonjs", + "moduleResolution": "node10", + "types": ["jest", "node"] + }, + "include": [ + "jest.config.ts", + "jest.config.cts", + "src/**/*.test.ts", + "src/**/*.spec.ts", + "src/**/*.d.ts" + ] +} diff --git a/libs/domains/loadweaver/shared/feature-cli-cluster/.eslintrc.json b/libs/domains/loadweaver/shared/feature-cli-cluster/.eslintrc.json new file mode 100644 index 000000000..b33995654 --- /dev/null +++ b/libs/domains/loadweaver/shared/feature-cli-cluster/.eslintrc.json @@ -0,0 +1,18 @@ +{ + "extends": ["../../../../../.eslintrc.json"], + "ignorePatterns": ["!**/*"], + "overrides": [ + { + "files": ["*.ts", "*.tsx", "*.js", "*.jsx"], + "rules": {} + }, + { + "files": ["*.ts", "*.tsx"], + "rules": {} + }, + { + "files": ["*.js", "*.jsx"], + "rules": {} + } + ] +} diff --git a/libs/domains/loadweaver/shared/feature-cli-cluster/README.md b/libs/domains/loadweaver/shared/feature-cli-cluster/README.md new file mode 100644 index 000000000..db95553ba --- /dev/null +++ b/libs/domains/loadweaver/shared/feature-cli-cluster/README.md @@ -0,0 +1,7 @@ +# loadweaver-shared-feature-cli-cluster + +This library was generated with [Nx](https://nx.dev). + +## Running unit tests + +Run `nx test loadweaver-shared-feature-cli-cluster` to execute the unit tests via [Jest](https://jestjs.io). diff --git a/libs/domains/loadweaver/shared/feature-cli-cluster/jest.config.cts b/libs/domains/loadweaver/shared/feature-cli-cluster/jest.config.cts new file mode 100644 index 000000000..2588ab8e8 --- /dev/null +++ b/libs/domains/loadweaver/shared/feature-cli-cluster/jest.config.cts @@ -0,0 +1,12 @@ +module.exports = { + displayName: 'loadweaver-shared-feature-cli-cluster', + preset: '../../../../../jest.preset.cjs', + testEnvironment: 'node', + transform: { + '^.+\\.[tj]s$': ['ts-jest', { tsconfig: '/tsconfig.spec.json' }], + '\\.tpl$': '/../../../../../tools/jest/tpl-loader.cjs', + }, + moduleFileExtensions: ['ts', 'js', 'html', 'tpl'], + coverageDirectory: + '../../../../../coverage/libs/domains/loadweaver/shared/feature-cli-cluster', +}; diff --git a/libs/domains/loadweaver/shared/feature-cli-cluster/project.json b/libs/domains/loadweaver/shared/feature-cli-cluster/project.json new file mode 100644 index 000000000..4566727a5 --- /dev/null +++ b/libs/domains/loadweaver/shared/feature-cli-cluster/project.json @@ -0,0 +1,16 @@ +{ + "name": "loadweaver-shared-feature-cli-cluster", + "$schema": "../../../../../node_modules/nx/schemas/project-schema.json", + "sourceRoot": "libs/domains/loadweaver/shared/feature-cli-cluster/src", + "projectType": "library", + "tags": ["domain:loadweaver", "scope:shared", "type:feature"], + "targets": { + "test": { + "executor": "@nx/jest:jest", + "outputs": ["{workspaceRoot}/coverage/{projectRoot}"], + "options": { + "jestConfig": "libs/domains/loadweaver/shared/feature-cli-cluster/jest.config.cts" + } + } + } +} diff --git a/libs/domains/loadweaver/shared/feature-cli-cluster/src/index.ts b/libs/domains/loadweaver/shared/feature-cli-cluster/src/index.ts new file mode 100644 index 000000000..bb63d9fed --- /dev/null +++ b/libs/domains/loadweaver/shared/feature-cli-cluster/src/index.ts @@ -0,0 +1,9 @@ +export { registerClusterCommands } from './lib/command'; +export { ClusterOrchestrator } from './lib/cluster-orchestrator'; +export { inspectRemoteDrift } from './lib/mutation-guard'; +export { + persistClusterInventory, + refreshInventoryFromLive, + acquireHostLocks, + releaseHostLocks, +} from './lib/node-inventory.service'; diff --git a/libs/domains/loadweaver/shared/feature-cli-cluster/src/lib/cluster-orchestrator.spec.ts b/libs/domains/loadweaver/shared/feature-cli-cluster/src/lib/cluster-orchestrator.spec.ts new file mode 100644 index 000000000..68da14e59 --- /dev/null +++ b/libs/domains/loadweaver/shared/feature-cli-cluster/src/lib/cluster-orchestrator.spec.ts @@ -0,0 +1,188 @@ +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; + +import { ClusterOrchestrator } from './cluster-orchestrator'; +import type { ClusterState } from './cluster-state'; +import type { LoadweaverContext } from '@forepath/loadweaver/shared/util-cli-core'; +import { defaultWorkspaceDir } from '@forepath/loadweaver/shared/util-cli-core'; + +function createMockContext(includeVip = false, includeRouting = false): LoadweaverContext { + return { + options: { + configPath: './loadweaver.yml', + dryRun: true, + verbose: false, + debug: false, + yes: false, + acceptDrift: false, + local: false, + json: false, + }, + config: { + version: 1, + cluster: { name: 'test', primaryManager: 'node-a1' }, + nodes: { + 'node-a1': { + hostname: 'a1.example.com', + wireguardIp: '10.200.0.1', + roles: ['manager', 'ceph-mon'], + }, + }, + wireguard: { + interface: 'wg0', + port: 51820, + mtu: 1420, + keyRotation: { enabled: false, intervalDays: 90, warnBeforeDays: 14 }, + }, + swarm: { advertiseInterface: 'wg0', overlayNetworks: ['traefik-public'] }, + ceph: { fsName: 'loadweaverfs', mountPath: '/mnt/cephfs', replication: 3, release: 'quincy' }, + traefik: { image: 'traefik:v3', network: 'traefik-public', mode: 'global' }, + volumes: [], + host: { configureFirewall: true }, + ...(includeVip + ? { + vip: { + address: '10.200.0.100', + interface: 'wg0', + backend: 'keepalived' as const, + routerId: 51, + pools: [], + }, + } + : {}), + ...(includeRouting + ? { + routing: { + enabled: true, + localAsn: 64512, + exportWireguardSubnet: true, + peers: [], + hubNodes: ['node-a1'], + }, + } + : {}), + }, + executor: { + run: async (command: string) => ({ command, stdout: '', stderr: '', exitCode: 0, dryRun: true }), + runScript: async () => [], + }, + logger: { + error: () => undefined, + warn: () => undefined, + info: () => undefined, + debug: () => undefined, + }, + sshForNode: () => ({ + execRemote: async (command: string) => ({ command, stdout: '', stderr: '', exitCode: 0, dryRun: true }), + uploadFile: async (command: string) => ({ command, stdout: '', stderr: '', exitCode: 0, dryRun: true }), + }), + sshTargetForNode: () => ({ host: '127.0.0.1', user: 'root' }), + }; +} + +describe('ClusterOrchestrator', () => { + it('defines init steps in dependency order without VIP when not configured', () => { + const steps = new ClusterOrchestrator(createMockContext()).initSteps().map((step) => step.name); + + expect(steps).toEqual([ + 'host.bootstrap', + 'prerequisites', + 'wireguard.init', + 'swarm.init', + 'swarm.join', + 'ceph.init', + 'ceph.cephfs-create', + 'ceph.cephfs-mount', + 'volume.create', + 'swarm.network.create', + 'traefik.deploy', + ]); + }); + + it('appends vip.init when VIP is configured', () => { + const steps = new ClusterOrchestrator(createMockContext(true)).initSteps().map((step) => step.name); + + expect(steps.at(-1)).toBe('vip.init'); + }); + + it('inserts routing.init after wireguard.init when routing is enabled', () => { + const steps = new ClusterOrchestrator(createMockContext(false, true)).initSteps().map((step) => step.name); + + expect(steps).toContain('routing.init'); + expect(steps.indexOf('routing.init')).toBe(steps.indexOf('wireguard.init') + 1); + }); + + it('defines destroy steps in reverse dependency order without VIP', () => { + const steps = new ClusterOrchestrator(createMockContext()).destroySteps().map((step) => step.name); + + expect(steps).toEqual(['traefik.destroy', 'swarm.leave', 'ceph.unmount', 'wireguard.teardown']); + }); + + it('inserts vip.destroy after traefik.destroy when VIP is configured', () => { + const steps = new ClusterOrchestrator(createMockContext(true)).destroySteps().map((step) => step.name); + + expect(steps).toEqual(['traefik.destroy', 'vip.destroy', 'swarm.leave', 'ceph.unmount', 'wireguard.teardown']); + }); + + it('prepends routing.destroy when routing is enabled', () => { + const steps = new ClusterOrchestrator(createMockContext(false, true)).destroySteps().map((step) => step.name); + + expect(steps[0]).toBe('routing.destroy'); + }); + + it('prepends wireguard.rotate-if-due when scheduled rotation is enabled and keys are due', () => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'loadweaver-orchestrator-')); + const configPath = path.join(tempDir, 'loadweaver.yml'); + fs.writeFileSync(configPath, 'version: 1\n'); + + const keysDir = path.join(defaultWorkspaceDir(configPath), 'wireguard'); + fs.mkdirSync(keysDir, { recursive: true }); + fs.writeFileSync( + path.join(keysDir, 'keys.json'), + JSON.stringify({ + version: 1, + nodes: { + 'node-a1': { privateKey: 'priv', publicKey: 'pub' }, + }, + }), + ); + + const ctx = createMockContext(); + ctx.options.configPath = configPath; + ctx.config!.wireguard.keyRotation = { enabled: true, intervalDays: 90, warnBeforeDays: 14 }; + + const previous: ClusterState = { + version: 1, + clusterName: 'test', + nodes: ['node-a1'], + swarmLabels: { + 'node-a1': ['loadweaver.role.manager=true'], + }, + traefikImage: 'traefik:v3', + traefikMode: 'global', + traefikAcmeEnabled: false, + traefikAcmeChallengeType: null, + traefikAcmeDnsProvider: null, + osdDevices: {}, + cephOsdNodes: [], + nodeHostnames: { 'node-a1': 'a1' }, + overlayNetworks: ['traefik-public'], + volumes: [], + vipConfigured: false, + routingEnabled: false, + routingHubNodes: [], + routingLocalAsn: null, + routingClusterCidr: null, + routingExportWireguardSubnet: true, + routingPeers: [], + updatedAt: '2026-01-01T00:00:00.000Z', + }; + + const steps = new ClusterOrchestrator(ctx).updateSteps(previous).map((step) => step.name); + + expect(steps[0]).toBe('wireguard.rotate-if-due'); + + fs.rmSync(tempDir, { recursive: true, force: true }); + }); +}); diff --git a/libs/domains/loadweaver/shared/feature-cli-cluster/src/lib/cluster-orchestrator.ts b/libs/domains/loadweaver/shared/feature-cli-cluster/src/lib/cluster-orchestrator.ts new file mode 100644 index 000000000..2441f1629 --- /dev/null +++ b/libs/domains/loadweaver/shared/feature-cli-cluster/src/lib/cluster-orchestrator.ts @@ -0,0 +1,384 @@ +import * as fs from 'node:fs'; + +import type { LoadweaverContext } from '@forepath/loadweaver/shared/util-cli-core'; +import { + assertPrerequisites, + printStructuredOutput, + runPrerequisiteChecks, +} from '@forepath/loadweaver/shared/util-cli-core'; + +import { CephService } from '@forepath/loadweaver/shared/feature-cli-ceph'; +import { HostService } from '@forepath/loadweaver/shared/feature-cli-host'; +import { NodeService } from '@forepath/loadweaver/shared/feature-cli-node'; +import { SwarmService } from '@forepath/loadweaver/shared/feature-cli-swarm'; +import { TraefikService } from '@forepath/loadweaver/shared/feature-cli-traefik'; +import { VipService } from '@forepath/loadweaver/shared/feature-cli-vip'; +import { RoutingService } from '@forepath/loadweaver/shared/feature-cli-routing'; +import { VolumeService } from '@forepath/loadweaver/shared/feature-cli-volume'; +import { WireguardService } from '@forepath/loadweaver/shared/feature-cli-wireguard'; + +import { planClusterUpdate, type UpdateAction } from './cluster-update-planner'; +import { defaultStatePath, loadClusterState, type ClusterState } from './cluster-state'; +import { runGuardedMutation } from '@forepath/loadweaver/shared/util-cli-core'; +import { removeHostInventories } from './node-inventory.service'; + +export type OrchestratorStep = { + name: string; + run: () => Promise; +}; + +type ClusterStatusSnapshot = { + prerequisites: Awaited>; + wireguard: { nodes: Array<{ nodeId: string; exitCode: number; output: string }> }; + swarm: { nodesOutput: string; networksOutput: string; nodesExitCode: number; networksExitCode: number }; + ceph: { exitCode: number; output: string }; + traefik: { exitCode: number; output: string }; + vip?: { nodes: Array<{ nodeId: string; exitCode: number; output: string }> }; + routing?: { hubs: Array<{ nodeId: string; exitCode: number; output: string }> }; +}; + +export class ClusterOrchestrator { + constructor(private readonly ctx: LoadweaverContext) {} + + initSteps(): OrchestratorStep[] { + const wireguard = new WireguardService(this.ctx); + const swarm = new SwarmService(this.ctx); + const ceph = new CephService(this.ctx); + const volume = new VolumeService(this.ctx); + const traefik = new TraefikService(this.ctx); + const vip = new VipService(this.ctx); + const routing = new RoutingService(this.ctx); + + const steps: OrchestratorStep[] = [ + { name: 'host.bootstrap', run: () => new HostService(this.ctx).bootstrapAll() }, + { + name: 'prerequisites', + run: async () => { + assertPrerequisites(await runPrerequisiteChecks(this.ctx)); + await new HostService(this.ctx).verifyAll(); + }, + }, + { name: 'wireguard.init', run: () => wireguard.init() }, + ]; + + if (this.ctx.config?.routing?.enabled) { + steps.push({ name: 'routing.init', run: () => routing.init() }); + } + + steps.push( + { name: 'swarm.init', run: () => swarm.init() }, + { name: 'swarm.join', run: () => swarm.joinAll() }, + { name: 'ceph.init', run: () => ceph.init() }, + { name: 'ceph.cephfs-create', run: () => ceph.createCephfs() }, + { name: 'ceph.cephfs-mount', run: () => ceph.mountCephfsAll() }, + { name: 'volume.create', run: () => volume.createAll() }, + { name: 'swarm.network.create', run: () => swarm.createNetworks() }, + { name: 'traefik.deploy', run: () => traefik.deploy() }, + ); + + if (this.ctx.config?.vip) { + steps.push({ name: 'vip.init', run: () => vip.init() }); + } + + return steps; + } + + updateSteps(previous: ClusterState | undefined): OrchestratorStep[] { + if (!previous) { + return this.initSteps(); + } + + if (!this.ctx.config) { + throw new Error('Configuration not loaded'); + } + + const actions = planClusterUpdate(previous, this.ctx.config, { + allowNodeRemoval: this.ctx.options.yes || this.ctx.options.dryRun, + }); + + if (actions.length === 0 && !this.shouldRotateWireguardKeys()) { + return [{ name: 'noop', run: async () => this.ctx.logger.info('Cluster is already converged') }]; + } + + const steps = actions.map((action) => this.toUpdateStep(action, previous)); + const rotationStep = this.wireguardRotationStep(); + + if (rotationStep) { + steps.unshift(rotationStep); + } + + return steps; + } + + private shouldRotateWireguardKeys(): boolean { + if (!this.ctx.config?.wireguard.keyRotation.enabled) { + return false; + } + + const evaluation = new WireguardService(this.ctx).evaluateRotation(); + + return evaluation.dueNodeIds.length > 0; + } + + private wireguardRotationStep(): OrchestratorStep | undefined { + if (!this.shouldRotateWireguardKeys()) { + return undefined; + } + + return { + name: 'wireguard.rotate-if-due', + run: () => new WireguardService(this.ctx).rotateIfDue().then(() => undefined), + }; + } + + destroySteps(): OrchestratorStep[] { + const wireguard = new WireguardService(this.ctx); + const swarm = new SwarmService(this.ctx); + const ceph = new CephService(this.ctx); + const traefik = new TraefikService(this.ctx); + const vip = new VipService(this.ctx); + const routing = new RoutingService(this.ctx); + + const steps: OrchestratorStep[] = [{ name: 'traefik.destroy', run: () => traefik.destroy() }]; + + if (this.ctx.config?.routing?.enabled) { + steps.unshift({ name: 'routing.destroy', run: () => routing.destroy() }); + } + + steps.push( + { name: 'swarm.leave', run: () => swarm.leaveAll() }, + { name: 'ceph.unmount', run: () => ceph.unmountAll() }, + { name: 'wireguard.teardown', run: () => wireguard.teardownAndClearKeys() }, + ); + + if (this.ctx.config?.vip) { + steps.splice(this.ctx.config?.routing?.enabled ? 2 : 1, 0, { name: 'vip.destroy', run: () => vip.destroy() }); + } + + return steps; + } + + async init(): Promise { + await runGuardedMutation(this.ctx, 'cluster.init', async () => { + await this.runSteps(this.initSteps(), 'cluster.init'); + }); + } + + async update(): Promise { + await runGuardedMutation(this.ctx, 'cluster.update', async () => { + const statePath = defaultStatePath(this.ctx.options.configPath); + const previous = loadClusterState(this.ctx.options.configPath); + + this.ctx.logger.info( + previous + ? `Applying incremental update from ${statePath}` + : 'No cluster state found; running full init before recording state', + ); + + await this.runSteps(this.updateSteps(previous), 'cluster.update'); + }); + } + + async destroy(): Promise { + await runGuardedMutation(this.ctx, 'cluster.destroy', async () => { + await this.runSteps(this.destroySteps(), 'cluster.destroy'); + await this.clearState(); + }); + } + + async status(): Promise { + const snapshot = await this.collectStatus(); + + if (this.ctx.options.json) { + printStructuredOutput(this.ctx, snapshot); + return; + } + + console.log('Prerequisites:', JSON.stringify(snapshot.prerequisites, null, 2)); + + for (const node of snapshot.wireguard.nodes) { + this.ctx.logger.info(`${node.nodeId}: exit=${node.exitCode}`); + if (node.output) { + console.log(node.output); + } + } + + if (snapshot.swarm.nodesOutput) { + console.log(snapshot.swarm.nodesOutput); + } + + if (snapshot.swarm.networksOutput) { + console.log(snapshot.swarm.networksOutput); + } + + if (snapshot.ceph.output) { + console.log(snapshot.ceph.output); + } + + if (snapshot.traefik.output) { + console.log(snapshot.traefik.output); + } + + if (snapshot.vip) { + for (const node of snapshot.vip.nodes) { + this.ctx.logger.info(`VIP status for ${node.nodeId}`); + if (node.output) { + console.log(node.output); + } + } + } + + if (snapshot.routing) { + for (const hub of snapshot.routing.hubs) { + this.ctx.logger.info(`Routing status for ${hub.nodeId}`); + if (hub.output) { + console.log(hub.output); + } + } + } + } + + private async collectStatus(): Promise { + const checks = await runPrerequisiteChecks(this.ctx); + const wireguard = new WireguardService(this.ctx); + const swarm = new SwarmService(this.ctx); + const ceph = new CephService(this.ctx); + const traefik = new TraefikService(this.ctx); + + const wireguardNodes = await wireguard.inspectStatus(); + const swarmStatus = await swarm.inspectStatus(); + const cephStatus = await ceph.inspectStatus(); + const traefikStatus = await traefik.inspectStatus(); + + const snapshot: ClusterStatusSnapshot = { + prerequisites: checks, + wireguard: { nodes: wireguardNodes }, + swarm: swarmStatus, + ceph: { + exitCode: cephStatus.exitCode, + output: cephStatus.output || cephStatus.stderr, + }, + traefik: traefikStatus, + }; + + if (this.ctx.config?.vip) { + snapshot.vip = { nodes: await new VipService(this.ctx).inspectStatus() }; + } + + if (this.ctx.config?.routing?.enabled) { + snapshot.routing = { hubs: await new RoutingService(this.ctx).inspectStatus() }; + } + + return snapshot; + } + + private toUpdateStep(action: UpdateAction, previous?: ClusterState): OrchestratorStep { + const wireguard = new WireguardService(this.ctx); + const swarm = new SwarmService(this.ctx); + const ceph = new CephService(this.ctx); + const volume = new VolumeService(this.ctx); + const traefik = new TraefikService(this.ctx); + const vip = new VipService(this.ctx); + const routing = new RoutingService(this.ctx); + const node = new NodeService(this.ctx); + + switch (action.type) { + case 'host.bootstrap': + return { + name: `host.bootstrap.${action.nodeId}`, + run: () => new HostService(this.ctx).bootstrapAndVerifyNode(action.nodeId), + }; + case 'wireguard.reconcile': + return { name: 'wireguard.reconcile', run: () => wireguard.reconcile() }; + case 'wireguard.add-peer': + return { name: `wireguard.add-peer.${action.nodeId}`, run: () => wireguard.addPeer(action.nodeId) }; + case 'wireguard.remove-peer': + return { name: `wireguard.remove-peer.${action.nodeId}`, run: () => wireguard.removePeer(action.nodeId) }; + case 'swarm.join': + return { name: 'swarm.join', run: () => swarm.joinNodes(action.nodeIds) }; + case 'swarm.reconcile-labels': + return { name: 'swarm.reconcile-labels', run: () => swarm.reconcileLabels(action.nodeIds) }; + case 'ceph.mount': + return { name: 'ceph.cephfs-mount', run: () => ceph.mountCephfs(action.nodeIds) }; + case 'ceph.osd-add': + return { name: `ceph.osd-add.${action.nodeId}`, run: () => ceph.addOsdForNode(action.nodeId) }; + case 'ceph.osd-remove': + return { + name: `ceph.osd-remove.${action.nodeId}`, + run: () => ceph.removeOsdsForNode(action.nodeId, action.hostname), + }; + case 'ceph.osd-reconcile': + return { name: 'ceph.osd-reconcile', run: () => ceph.reconcileOsds() }; + case 'volume.create': + return { name: 'volume.create', run: () => volume.createVolumes(action.volumeNames) }; + case 'swarm.network.create': + return { name: 'swarm.network.create', run: () => swarm.createNetworks(action.networkNames) }; + case 'traefik.update': + return { name: 'traefik.update', run: () => traefik.update() }; + case 'vip.init': + return { name: 'vip.init', run: () => vip.init() }; + case 'vip.reconcile': + return { name: 'vip.reconcile', run: () => vip.reconcile() }; + case 'vip.destroy': + return { name: 'vip.destroy', run: () => vip.destroy() }; + case 'routing.init': + return { name: 'routing.init', run: () => routing.init() }; + case 'routing.reconcile': + return { name: 'routing.reconcile', run: () => routing.reconcile() }; + case 'routing.destroy': + return { name: 'routing.destroy', run: () => routing.destroy() }; + case 'node.leave': + return { + name: `node.leave.${action.nodeId}`, + run: () => + node.leave(action.nodeId, { + hostname: previous?.nodeHostnames?.[action.nodeId], + skipOsdRemoval: Boolean( + previous?.nodeHostnames?.[action.nodeId] && + (previous?.osdDevices?.[action.nodeId] || previous?.cephOsdNodes?.includes(action.nodeId)), + ), + }), + }; + default: { + const exhaustive: never = action; + throw new Error(`Unsupported update action: ${JSON.stringify(exhaustive)}`); + } + } + } + + private async clearState(): Promise { + if (this.ctx.options.dryRun) { + return; + } + + await removeHostInventories(this.ctx); + + const statePath = defaultStatePath(this.ctx.options.configPath); + + if (fs.existsSync(statePath)) { + fs.unlinkSync(statePath); + } + } + + private async runSteps(steps: OrchestratorStep[], operation: string): Promise { + const executed: string[] = []; + + for (const step of steps) { + executed.push(step.name); + + if (!(this.ctx.options.json && this.ctx.options.dryRun)) { + this.ctx.logger.info(`Running step: ${step.name}`); + } + + await step.run(); + } + + if (this.ctx.options.json && this.ctx.options.dryRun) { + printStructuredOutput(this.ctx, { + operation, + dryRun: true, + steps: executed, + }); + } + } +} diff --git a/libs/domains/loadweaver/shared/feature-cli-cluster/src/lib/cluster-state.spec.ts b/libs/domains/loadweaver/shared/feature-cli-cluster/src/lib/cluster-state.spec.ts new file mode 100644 index 000000000..78677475f --- /dev/null +++ b/libs/domains/loadweaver/shared/feature-cli-cluster/src/lib/cluster-state.spec.ts @@ -0,0 +1,47 @@ +import type { LoadweaverConfig } from '@forepath/loadweaver/shared/util-cli-core'; + +import { deriveStateFromConfig } from './cluster-state'; +import { AUTH_PASS_REDACTED } from './sanitize-desired-config'; + +const config: LoadweaverConfig = { + version: 1, + cluster: { name: 'prod', primaryManager: 'node-a1' }, + nodes: { + 'node-a1': { hostname: 'a1', wireguardIp: '10.200.0.1', roles: ['manager'] }, + }, + wireguard: { + interface: 'wg0', + port: 51820, + mtu: 1420, + keyRotation: { enabled: false, intervalDays: 90, warnBeforeDays: 14 }, + }, + swarm: { advertiseInterface: 'wg0', overlayNetworks: ['traefik-public'] }, + ceph: { fsName: 'fs', mountPath: '/mnt/cephfs', replication: 3, release: 'quincy' }, + traefik: { image: 'traefik:v3', network: 'traefik-public', mode: 'global' }, + host: { configureFirewall: true }, + volumes: [], + vip: { + address: '10.200.0.100', + interface: 'eth0', + backend: 'keepalived', + authPass: 'secret01', + pools: [], + }, +}; + +describe('deriveStateFromConfig', () => { + it('starts inventory serial at 1 and stores a sanitized desired config', () => { + const state = deriveStateFromConfig(config); + + expect(state.inventorySerial).toBe(1); + expect(state.desired?.vip?.authPass).toBe(AUTH_PASS_REDACTED); + expect(state.nodes).toEqual(['node-a1']); + }); + + it('increments inventory serial from previous state', () => { + const previous = deriveStateFromConfig(config); + const next = deriveStateFromConfig(config, undefined, { previous, bumpSerial: true }); + + expect(next.inventorySerial).toBe(2); + }); +}); diff --git a/libs/domains/loadweaver/shared/feature-cli-cluster/src/lib/cluster-state.ts b/libs/domains/loadweaver/shared/feature-cli-cluster/src/lib/cluster-state.ts new file mode 100644 index 000000000..f557609b9 --- /dev/null +++ b/libs/domains/loadweaver/shared/feature-cli-cluster/src/lib/cluster-state.ts @@ -0,0 +1,117 @@ +import * as fs from 'node:fs'; +import * as path from 'node:path'; + +import type { LoadweaverConfig } from '@forepath/loadweaver/shared/util-cli-core'; +import { + clusterStatePath, + deriveExpectedSwarmLabels, + deriveOsdDevices, + deriveRoutingStateSnapshot, + deriveVipStateSnapshot, +} from '@forepath/loadweaver/shared/util-cli-core'; + +import type { RemoteFingerprint } from './remote-fingerprint'; +import { sanitizeDesiredConfig } from './sanitize-desired-config'; + +export interface DeriveStateOptions { + previous?: ClusterState; + bumpSerial?: boolean; + keepDesired?: boolean; +} + +export interface ClusterState { + version: number; + clusterName: string; + nodes: string[]; + swarmLabels: Record; + traefikImage: string; + traefikMode: string; + traefikAcmeEnabled: boolean; + traefikAcmeChallengeType: 'http' | 'dns' | null; + traefikAcmeDnsProvider: string | null; + osdDevices: Record; + cephOsdNodes: string[]; + overlayNetworks: string[]; + volumes: string[]; + vipConfigured: boolean; + vipFingerprint?: string; + routingEnabled: boolean; + routingHubNodes: string[]; + routingLocalAsn: number | null; + routingClusterCidr: string | null; + routingExportWireguardSubnet: boolean; + routingPeers: Array<{ name: string; remoteAsn: number; neighbor: string }>; + nodeHostnames: Record; + updatedAt: string; + inventorySerial?: number; + desired?: LoadweaverConfig; + remoteFingerprint?: RemoteFingerprint; +} + +export function defaultStatePath(configPath: string): string { + return clusterStatePath(configPath); +} + +export function deriveStateFromConfig( + config: LoadweaverConfig, + remoteFingerprint?: RemoteFingerprint, + options: DeriveStateOptions = {}, +): ClusterState { + const routing = deriveRoutingStateSnapshot(config); + const vip = deriveVipStateSnapshot(config); + const previousSerial = options.previous?.inventorySerial ?? 0; + const bumpSerial = options.bumpSerial !== false; + const inventorySerial = bumpSerial ? previousSerial + 1 : previousSerial || undefined; + const desired = options.keepDesired + ? (options.previous?.desired ?? sanitizeDesiredConfig(config)) + : sanitizeDesiredConfig(config); + + return { + version: config.version, + clusterName: config.cluster.name, + nodes: Object.keys(config.nodes).sort(), + swarmLabels: deriveExpectedSwarmLabels(config), + traefikImage: config.traefik.image, + traefikMode: config.traefik.mode, + traefikAcmeEnabled: Boolean(config.traefik.acme), + traefikAcmeChallengeType: config.traefik.acme?.challengeType ?? null, + traefikAcmeDnsProvider: config.traefik.acme?.dnsProvider ?? null, + osdDevices: deriveOsdDevices(config), + cephOsdNodes: Object.keys(config.nodes) + .filter((nodeId) => config.nodes[nodeId].roles.includes('ceph-osd')) + .sort(), + overlayNetworks: [...config.swarm.overlayNetworks].sort(), + volumes: config.volumes.map((volume) => volume.name).sort(), + vipConfigured: vip.configured, + vipFingerprint: vip.fingerprint || undefined, + routingEnabled: routing.enabled, + routingHubNodes: routing.hubNodes, + routingLocalAsn: routing.localAsn, + routingClusterCidr: routing.clusterCidr, + routingExportWireguardSubnet: routing.exportWireguardSubnet, + routingPeers: routing.peers, + nodeHostnames: Object.fromEntries(Object.entries(config.nodes).map(([nodeId, node]) => [nodeId, node.hostname])), + updatedAt: new Date().toISOString(), + ...(inventorySerial !== undefined ? { inventorySerial } : {}), + ...(desired ? { desired } : {}), + ...(remoteFingerprint ? { remoteFingerprint } : {}), + }; +} + +export function loadClusterState(configPath: string): ClusterState | undefined { + const absolutePath = clusterStatePath(configPath); + + if (!fs.existsSync(absolutePath)) { + return undefined; + } + + return JSON.parse(fs.readFileSync(absolutePath, 'utf-8')) as ClusterState; +} + +export function saveClusterState(configPath: string, state: ClusterState): void { + const absolutePath = clusterStatePath(configPath); + fs.mkdirSync(path.dirname(absolutePath), { recursive: true }); + fs.writeFileSync(absolutePath, `${JSON.stringify(state, null, 2)}\n`, 'utf-8'); +} + +export { clusterStatePath }; diff --git a/libs/domains/loadweaver/shared/feature-cli-cluster/src/lib/cluster-update-planner.spec.ts b/libs/domains/loadweaver/shared/feature-cli-cluster/src/lib/cluster-update-planner.spec.ts new file mode 100644 index 000000000..21636df48 --- /dev/null +++ b/libs/domains/loadweaver/shared/feature-cli-cluster/src/lib/cluster-update-planner.spec.ts @@ -0,0 +1,311 @@ +import { planClusterUpdate } from './cluster-update-planner'; +import type { ClusterState } from './cluster-state'; + +const baseState: ClusterState = { + version: 1, + clusterName: 'prod', + nodes: ['node-a1', 'node-a2'], + swarmLabels: { + 'node-a1': ['loadweaver.role.manager=true'], + 'node-a2': ['loadweaver.role.worker=true'], + }, + traefikImage: 'traefik:v3', + traefikMode: 'global', + traefikAcmeEnabled: false, + traefikAcmeChallengeType: null, + traefikAcmeDnsProvider: null, + osdDevices: {}, + cephOsdNodes: [], + nodeHostnames: { + 'node-a1': 'a1', + 'node-a2': 'a2', + }, + overlayNetworks: ['traefik-public'], + volumes: ['traefik-config'], + vipConfigured: false, + routingEnabled: false, + routingHubNodes: [], + routingLocalAsn: null, + routingClusterCidr: null, + routingExportWireguardSubnet: true, + routingPeers: [], + updatedAt: '2026-01-01T00:00:00.000Z', +}; + +describe('planClusterUpdate', () => { + it('plans peer, swarm, mount, and volume steps for newly added nodes', () => { + const actions = planClusterUpdate( + baseState, + { + version: 1, + cluster: { name: 'prod', primaryManager: 'node-a1' }, + nodes: { + 'node-a1': { hostname: 'a1', wireguardIp: '10.200.0.1', roles: ['manager'] }, + 'node-a2': { hostname: 'a2', wireguardIp: '10.200.0.2', roles: ['worker'] }, + 'node-a3': { hostname: 'a3', wireguardIp: '10.200.0.3', roles: ['worker'] }, + }, + wireguard: { + interface: 'wg0', + port: 51820, + mtu: 1420, + keyRotation: { enabled: false, intervalDays: 90, warnBeforeDays: 14 }, + }, + swarm: { advertiseInterface: 'wg0', overlayNetworks: ['traefik-public'] }, + ceph: { fsName: 'fs', mountPath: '/mnt/cephfs', replication: 3, release: 'quincy' }, + traefik: { image: 'traefik:v3', network: 'traefik-public', mode: 'global' }, + host: { configureFirewall: true }, + volumes: [{ name: 'traefik-config', path: 'traefik/config' }], + }, + { allowNodeRemoval: false }, + ); + + expect(actions.map((action) => action.type)).toEqual([ + 'host.bootstrap', + 'wireguard.reconcile', + 'wireguard.add-peer', + 'swarm.join', + 'ceph.mount', + ]); + }); + + it('plans traefik update when image changes', () => { + const actions = planClusterUpdate( + baseState, + { + version: 1, + cluster: { name: 'prod', primaryManager: 'node-a1' }, + nodes: { + 'node-a1': { hostname: 'a1', wireguardIp: '10.200.0.1', roles: ['manager'] }, + 'node-a2': { hostname: 'a2', wireguardIp: '10.200.0.2', roles: ['worker'] }, + }, + wireguard: { + interface: 'wg0', + port: 51820, + mtu: 1420, + keyRotation: { enabled: false, intervalDays: 90, warnBeforeDays: 14 }, + }, + swarm: { advertiseInterface: 'wg0', overlayNetworks: ['traefik-public'] }, + ceph: { fsName: 'fs', mountPath: '/mnt/cephfs', replication: 3, release: 'quincy' }, + traefik: { image: 'traefik:v3.1', network: 'traefik-public', mode: 'global' }, + host: { configureFirewall: true }, + volumes: [{ name: 'traefik-config', path: 'traefik/config' }], + }, + { allowNodeRemoval: false }, + ); + + expect(actions).toEqual([{ type: 'traefik.update' }]); + }); + + it('plans swarm.reconcile-labels when node roles change without membership changes', () => { + const actions = planClusterUpdate( + baseState, + { + version: 1, + cluster: { name: 'prod', primaryManager: 'node-a1' }, + nodes: { + 'node-a1': { hostname: 'a1', wireguardIp: '10.200.0.1', roles: ['manager'] }, + 'node-a2': { hostname: 'a2', wireguardIp: '10.200.0.2', roles: ['manager', 'worker'] }, + }, + wireguard: { + interface: 'wg0', + port: 51820, + mtu: 1420, + keyRotation: { enabled: false, intervalDays: 90, warnBeforeDays: 14 }, + }, + swarm: { advertiseInterface: 'wg0', overlayNetworks: ['traefik-public'] }, + ceph: { fsName: 'fs', mountPath: '/mnt/cephfs', replication: 3, release: 'quincy' }, + traefik: { image: 'traefik:v3', network: 'traefik-public', mode: 'global' }, + host: { configureFirewall: true }, + volumes: [{ name: 'traefik-config', path: 'traefik/config' }], + }, + { allowNodeRemoval: false }, + ); + + expect(actions).toEqual([{ type: 'swarm.reconcile-labels', nodeIds: ['node-a2'] }]); + }); + + it('plans routing init when routing is newly enabled', () => { + const actions = planClusterUpdate( + baseState, + { + version: 1, + cluster: { name: 'prod', primaryManager: 'node-a1' }, + nodes: { + 'node-a1': { hostname: 'a1', wireguardIp: '10.200.0.1', roles: ['manager'] }, + 'node-a2': { hostname: 'a2', wireguardIp: '10.200.0.2', roles: ['worker'] }, + }, + wireguard: { + interface: 'wg0', + port: 51820, + mtu: 1420, + keyRotation: { enabled: false, intervalDays: 90, warnBeforeDays: 14 }, + }, + swarm: { advertiseInterface: 'wg0', overlayNetworks: ['traefik-public'] }, + ceph: { fsName: 'fs', mountPath: '/mnt/cephfs', replication: 3, release: 'quincy' }, + traefik: { image: 'traefik:v3', network: 'traefik-public', mode: 'global' }, + host: { configureFirewall: true }, + volumes: [{ name: 'traefik-config', path: 'traefik/config' }], + routing: { + enabled: true, + localAsn: 64512, + exportWireguardSubnet: true, + peers: [], + hubNodes: ['node-a1'], + }, + }, + { allowNodeRemoval: false }, + ); + + expect(actions).toEqual([{ type: 'routing.init' }, { type: 'swarm.reconcile-labels', nodeIds: ['node-a1'] }]); + }); + + it('plans ceph osd-add when osdDevice is added to an existing node', () => { + const actions = planClusterUpdate( + { + ...baseState, + osdDevices: { 'node-a2': '/dev/sdb' }, + swarmLabels: { + 'node-a1': ['loadweaver.role.manager=true'], + 'node-a2': ['loadweaver.role.ceph-osd=true', 'loadweaver.role.worker=true'], + }, + }, + { + version: 1, + cluster: { name: 'prod', primaryManager: 'node-a1' }, + nodes: { + 'node-a1': { hostname: 'a1', wireguardIp: '10.200.0.1', roles: ['manager'] }, + 'node-a2': { + hostname: 'a2', + wireguardIp: '10.200.0.2', + roles: ['worker', 'ceph-osd'], + osdDevice: '/dev/sdc', + }, + }, + wireguard: { + interface: 'wg0', + port: 51820, + mtu: 1420, + keyRotation: { enabled: false, intervalDays: 90, warnBeforeDays: 14 }, + }, + swarm: { advertiseInterface: 'wg0', overlayNetworks: ['traefik-public'] }, + ceph: { fsName: 'fs', mountPath: '/mnt/cephfs', replication: 3, release: 'quincy' }, + traefik: { image: 'traefik:v3', network: 'traefik-public', mode: 'global' }, + host: { configureFirewall: true }, + volumes: [{ name: 'traefik-config', path: 'traefik/config' }], + }, + { allowNodeRemoval: false }, + ); + + expect(actions).toEqual([{ type: 'ceph.osd-add', nodeId: 'node-a2' }]); + }); + + it('plans ceph osd-remove before node leave when osd node is removed', () => { + const actions = planClusterUpdate( + { + ...baseState, + osdDevices: { 'node-a2': '/dev/sdb' }, + cephOsdNodes: ['node-a2'], + nodeHostnames: { 'node-a1': 'a1', 'node-a2': 'a2' }, + }, + { + version: 1, + cluster: { name: 'prod', primaryManager: 'node-a1' }, + nodes: { + 'node-a1': { hostname: 'a1', wireguardIp: '10.200.0.1', roles: ['manager'] }, + }, + wireguard: { + interface: 'wg0', + port: 51820, + mtu: 1420, + keyRotation: { enabled: false, intervalDays: 90, warnBeforeDays: 14 }, + }, + swarm: { advertiseInterface: 'wg0', overlayNetworks: ['traefik-public'] }, + ceph: { fsName: 'fs', mountPath: '/mnt/cephfs', replication: 3, release: 'quincy' }, + traefik: { image: 'traefik:v3', network: 'traefik-public', mode: 'global' }, + host: { configureFirewall: true }, + volumes: [{ name: 'traefik-config', path: 'traefik/config' }], + }, + { allowNodeRemoval: true }, + ); + + expect(actions.map((action) => action.type)).toEqual(['ceph.osd-remove', 'node.leave', 'wireguard.remove-peer']); + }); + + it('requires confirmation before removing nodes', () => { + expect(() => + planClusterUpdate( + baseState, + { + version: 1, + cluster: { name: 'prod', primaryManager: 'node-a1' }, + nodes: { + 'node-a1': { hostname: 'a1', wireguardIp: '10.200.0.1', roles: ['manager'] }, + }, + wireguard: { + interface: 'wg0', + port: 51820, + mtu: 1420, + keyRotation: { enabled: false, intervalDays: 90, warnBeforeDays: 14 }, + }, + swarm: { advertiseInterface: 'wg0', overlayNetworks: ['traefik-public'] }, + ceph: { fsName: 'fs', mountPath: '/mnt/cephfs', replication: 3, release: 'quincy' }, + traefik: { image: 'traefik:v3', network: 'traefik-public', mode: 'global' }, + host: { configureFirewall: true }, + volumes: [{ name: 'traefik-config', path: 'traefik/config' }], + }, + { allowNodeRemoval: false }, + ), + ).toThrow(/--yes/); + }); + + it('plans vip.reconcile when vip fingerprint changes', () => { + const actions = planClusterUpdate( + { + ...baseState, + vipConfigured: true, + vipFingerprint: 'previous', + }, + { + version: 1, + cluster: { name: 'prod', primaryManager: 'node-a1' }, + nodes: { + 'node-a1': { hostname: 'a1', wireguardIp: '10.200.0.1', roles: ['manager'] }, + 'node-a2': { hostname: 'a2', wireguardIp: '10.200.0.2', roles: ['worker'] }, + }, + wireguard: { + interface: 'wg0', + port: 51820, + mtu: 1420, + keyRotation: { enabled: false, intervalDays: 90, warnBeforeDays: 14 }, + }, + swarm: { advertiseInterface: 'wg0', overlayNetworks: ['traefik-public'] }, + ceph: { fsName: 'fs', mountPath: '/mnt/cephfs', replication: 3, release: 'quincy' }, + traefik: { image: 'traefik:v3', network: 'traefik-public', mode: 'global' }, + host: { configureFirewall: true }, + volumes: [{ name: 'traefik-config', path: 'traefik/config' }], + vip: { + address: '203.0.113.100', + interface: 'eth0', + backend: 'keepalived', + pools: [ + { + name: 'postgres', + address: '203.0.113.101', + healthCheck: { type: 'tcp', path: '/' }, + listeners: [ + { + port: 5432, + protocol: 'tcp', + backends: [{ type: 'host', host: '10.200.0.50', port: 5432 }], + }, + ], + }, + ], + }, + }, + { allowNodeRemoval: false }, + ); + + expect(actions).toEqual([{ type: 'vip.reconcile' }]); + }); +}); diff --git a/libs/domains/loadweaver/shared/feature-cli-cluster/src/lib/cluster-update-planner.ts b/libs/domains/loadweaver/shared/feature-cli-cluster/src/lib/cluster-update-planner.ts new file mode 100644 index 000000000..7baa2d2af --- /dev/null +++ b/libs/domains/loadweaver/shared/feature-cli-cluster/src/lib/cluster-update-planner.ts @@ -0,0 +1,193 @@ +import type { LoadweaverConfig } from '@forepath/loadweaver/shared/util-cli-core'; +import { + diffOsdDeviceChanges, + diffSwarmLabelChanges, + deriveOsdDevices, + deriveRoutingStateSnapshot, + deriveVipStateSnapshot, + routingStateChanged, + vipStateChanged, +} from '@forepath/loadweaver/shared/util-cli-core'; + +import type { ClusterState } from './cluster-state'; + +export type UpdateAction = + | { type: 'host.bootstrap'; nodeId: string } + | { type: 'wireguard.reconcile' } + | { type: 'wireguard.add-peer'; nodeId: string } + | { type: 'wireguard.remove-peer'; nodeId: string } + | { type: 'swarm.join'; nodeIds: string[] } + | { type: 'swarm.reconcile-labels'; nodeIds: string[] } + | { type: 'ceph.mount'; nodeIds: string[] } + | { type: 'ceph.osd-add'; nodeId: string } + | { type: 'ceph.osd-remove'; nodeId: string; hostname: string } + | { type: 'ceph.osd-reconcile' } + | { type: 'volume.create'; volumeNames: string[] } + | { type: 'swarm.network.create'; networkNames: string[] } + | { type: 'traefik.update' } + | { type: 'vip.init' } + | { type: 'vip.reconcile' } + | { type: 'vip.destroy' } + | { type: 'routing.init' } + | { type: 'routing.reconcile' } + | { type: 'routing.destroy' } + | { type: 'node.leave'; nodeId: string }; + +function diffAdded(previous: string[], current: string[]): string[] { + const previousSet = new Set(previous); + return current.filter((item) => !previousSet.has(item)); +} + +function diffRemoved(previous: string[], current: string[]): string[] { + const currentSet = new Set(current); + return previous.filter((item) => !currentSet.has(item)); +} + +export function planClusterUpdate( + previous: ClusterState | undefined, + config: LoadweaverConfig, + options: { allowNodeRemoval: boolean }, +): UpdateAction[] { + if (!previous) { + return []; + } + + const current = { + nodes: Object.keys(config.nodes).sort(), + traefikImage: config.traefik.image, + traefikMode: config.traefik.mode, + traefikAcmeEnabled: Boolean(config.traefik.acme), + traefikAcmeChallengeType: config.traefik.acme?.challengeType ?? null, + traefikAcmeDnsProvider: config.traefik.acme?.dnsProvider ?? null, + osdDevices: deriveOsdDevices(config), + overlayNetworks: [...config.swarm.overlayNetworks].sort(), + volumes: config.volumes.map((volume) => volume.name).sort(), + vipConfigured: Boolean(config.vip), + vipFingerprint: deriveVipStateSnapshot(config).fingerprint, + routing: deriveRoutingStateSnapshot(config), + }; + + const actions: UpdateAction[] = []; + const addedNodes = diffAdded(previous.nodes, current.nodes); + const removedNodes = diffRemoved(previous.nodes, current.nodes); + + if (addedNodes.length > 0) { + for (const nodeId of addedNodes) { + actions.push({ type: 'host.bootstrap', nodeId }); + } + + actions.push({ type: 'wireguard.reconcile' }); + + for (const nodeId of addedNodes) { + actions.push({ type: 'wireguard.add-peer', nodeId }); + } + + actions.push({ type: 'swarm.join', nodeIds: addedNodes }); + actions.push({ type: 'ceph.mount', nodeIds: addedNodes }); + + for (const nodeId of addedNodes) { + if (current.osdDevices[nodeId]) { + actions.push({ type: 'ceph.osd-add', nodeId }); + } + } + } + + if (removedNodes.length > 0) { + if (!options.allowNodeRemoval) { + throw new Error( + `Configuration removes node(s): ${removedNodes.join(', ')}. Re-run with --yes to apply node removal.`, + ); + } + + for (const nodeId of removedNodes) { + const hostname = previous.nodeHostnames?.[nodeId]; + + if (hostname && (previous.osdDevices?.[nodeId] || previous.cephOsdNodes?.includes(nodeId))) { + actions.push({ type: 'ceph.osd-remove', nodeId, hostname }); + } + + actions.push({ type: 'node.leave', nodeId }); + actions.push({ type: 'wireguard.remove-peer', nodeId }); + } + } + + const addedVolumes = diffAdded(previous.volumes, current.volumes); + + if (addedVolumes.length > 0) { + actions.push({ type: 'volume.create', volumeNames: addedVolumes }); + } + + const addedNetworks = diffAdded(previous.overlayNetworks, current.overlayNetworks); + + if (addedNetworks.length > 0) { + actions.push({ type: 'swarm.network.create', networkNames: addedNetworks }); + } + + const traefikChanged = + previous.traefikImage !== current.traefikImage || + previous.traefikMode !== current.traefikMode || + previous.traefikAcmeEnabled !== current.traefikAcmeEnabled || + (previous.traefikAcmeChallengeType ?? null) !== current.traefikAcmeChallengeType || + (previous.traefikAcmeDnsProvider ?? null) !== current.traefikAcmeDnsProvider; + + if (traefikChanged) { + actions.push({ type: 'traefik.update' }); + } + + if (!previous.vipConfigured && current.vipConfigured) { + actions.push({ type: 'vip.init' }); + } + + if (previous.vipConfigured && !current.vipConfigured) { + actions.push({ type: 'vip.destroy' }); + } + + if ( + previous.vipConfigured && + current.vipConfigured && + vipStateChanged(previous.vipFingerprint, deriveVipStateSnapshot(config)) + ) { + actions.push({ type: 'vip.reconcile' }); + } else if (previous.vipConfigured && current.vipConfigured && deriveVipStateSnapshot(config).hasSwarmBackends) { + actions.push({ type: 'vip.reconcile' }); + } + + const previousRouting = { + enabled: previous.routingEnabled, + hubNodes: previous.routingHubNodes, + localAsn: previous.routingLocalAsn, + clusterCidr: previous.routingClusterCidr, + exportWireguardSubnet: previous.routingExportWireguardSubnet, + peers: previous.routingPeers, + }; + + if (!previousRouting.enabled && current.routing.enabled) { + actions.push({ type: 'routing.init' }); + } + + if (previousRouting.enabled && !current.routing.enabled) { + actions.push({ type: 'routing.destroy' }); + } + + if (previousRouting.enabled && current.routing.enabled && routingStateChanged(previousRouting, current.routing)) { + actions.push({ type: 'routing.reconcile' }); + } + + const labelChangedNodes = diffSwarmLabelChanges(previous.swarmLabels, config).filter( + (nodeId) => !addedNodes.includes(nodeId), + ); + + if (labelChangedNodes.length > 0) { + actions.push({ type: 'swarm.reconcile-labels', nodeIds: labelChangedNodes }); + } + + const osdChangedNodes = diffOsdDeviceChanges(previous.osdDevices ?? {}, config).filter( + (nodeId) => !addedNodes.includes(nodeId), + ); + + for (const nodeId of osdChangedNodes) { + actions.push({ type: 'ceph.osd-add', nodeId }); + } + + return actions; +} diff --git a/libs/domains/loadweaver/shared/feature-cli-cluster/src/lib/command.ts b/libs/domains/loadweaver/shared/feature-cli-cluster/src/lib/command.ts new file mode 100644 index 000000000..d85aa1faf --- /dev/null +++ b/libs/domains/loadweaver/shared/feature-cli-cluster/src/lib/command.ts @@ -0,0 +1,79 @@ +import type { Command } from 'commander'; + +import type { LoadweaverContext } from '@forepath/loadweaver/shared/util-cli-core'; +import { printStructuredOutput, withExamples } from '@forepath/loadweaver/shared/util-cli-core'; + +import { ClusterOrchestrator } from './cluster-orchestrator'; +import { inspectRemoteDrift } from './mutation-guard'; + +export function registerClusterCommands(program: Command, getCtx: (command: Command) => LoadweaverContext): void { + const cluster = program.command('cluster').description('Cluster-wide lifecycle operations'); + withExamples(cluster, [ + 'loadweaver --config ./loadweaver.yml cluster status', + 'loadweaver --dry-run cluster init', + 'loadweaver --yes cluster destroy', + ]); + + const init = cluster + .command('init') + .description('Bootstrap the full stack according to configuration') + .action(async function (this: Command) { + await new ClusterOrchestrator(getCtx(this)).init(); + }); + withExamples(init, ['loadweaver cluster init', 'loadweaver --dry-run --verbose cluster init']); + + const update = cluster + .command('update') + .description('Apply configuration changes to the cluster') + .action(async function (this: Command) { + await new ClusterOrchestrator(getCtx(this)).update(); + }); + withExamples(update, ['loadweaver cluster update', 'loadweaver --dry-run cluster update']); + + const destroy = cluster + .command('destroy') + .description('Tear down the cluster stack') + .action(async function (this: Command) { + const ctx = getCtx(this); + + if (!ctx.options.yes && !ctx.options.dryRun) { + throw new Error('Refusing to destroy without --yes (or use --dry-run)'); + } + + await new ClusterOrchestrator(ctx).destroy(); + }); + withExamples(destroy, ['loadweaver --dry-run cluster destroy', 'loadweaver --yes cluster destroy']); + + const status = cluster + .command('status') + .description('Summarize cluster health') + .action(async function (this: Command) { + await new ClusterOrchestrator(getCtx(this)).status(); + }); + withExamples(status, [ + 'loadweaver cluster status', + 'loadweaver --json cluster status', + 'loadweaver --config ./loadweaver.yml cluster status', + ]); + + const drift = cluster + .command('drift') + .description('Compare last recorded inventory against the live cluster') + .action(async function (this: Command) { + const ctx = getCtx(this); + const findings = await inspectRemoteDrift(ctx); + + if (findings.length === 0) { + const payload = { drift: [], message: 'No remote drift detected (or no remote fingerprint baseline yet).' }; + if (ctx.options.json) { + printStructuredOutput(ctx, payload); + } else { + ctx.logger.info(payload.message); + } + return; + } + + printStructuredOutput(ctx, { drift: findings }); + }); + withExamples(drift, ['loadweaver cluster drift', 'loadweaver --verbose cluster drift']); +} diff --git a/libs/domains/loadweaver/shared/feature-cli-cluster/src/lib/drift-detector.spec.ts b/libs/domains/loadweaver/shared/feature-cli-cluster/src/lib/drift-detector.spec.ts new file mode 100644 index 000000000..ba279ad1b --- /dev/null +++ b/libs/domains/loadweaver/shared/feature-cli-cluster/src/lib/drift-detector.spec.ts @@ -0,0 +1,125 @@ +import type { LoadweaverConfig } from '@forepath/loadweaver/shared/util-cli-core'; + +import { detectRemoteDrift } from './drift-detector'; +import type { RemoteFingerprint } from './remote-fingerprint'; + +const config: LoadweaverConfig = { + version: 1, + cluster: { name: 'test', primaryManager: 'node-a1' }, + nodes: { + 'node-a1': { hostname: 'a1.example.com', wireguardIp: '10.200.0.1', roles: ['manager'] }, + 'node-a2': { hostname: 'a2.example.com', wireguardIp: '10.200.0.2', roles: ['worker'] }, + }, + wireguard: { + interface: 'wg0', + port: 51820, + mtu: 1420, + keyRotation: { enabled: false, intervalDays: 90, warnBeforeDays: 14 }, + }, + swarm: { advertiseInterface: 'wg0', overlayNetworks: ['traefik-public'] }, + ceph: { fsName: 'fs', mountPath: '/mnt/cephfs', replication: 3, release: 'quincy' }, + traefik: { image: 'traefik:v3', network: 'traefik-public', mode: 'global' as const }, + host: { configureFirewall: true }, + volumes: [], +}; + +function fingerprint(overrides: Partial = {}): RemoteFingerprint { + return { + capturedAt: '2026-01-01T00:00:00.000Z', + swarmNodeHostnames: ['a1.example.com', 'a2.example.com'], + swarmNodeLabels: { + 'node-a1': ['site=a'], + 'node-a2': ['site=a'], + }, + traefikImage: 'traefik:v3', + traefikDeployed: true, + traefikServiceVersion: '42', + traefikReplicas: 'traefik_traefik:2/2', + vipKeepalivedActive: false, + vipHolderNodeId: null, + vipHolders: {}, + cephHealth: 'HEALTH_OK', + cephMonitorCount: 3, + nodes: { + 'node-a1': { wireguardActive: true, wireguardPeerCount: 1, swarmActive: true, cephMounted: true }, + 'node-a2': { wireguardActive: true, wireguardPeerCount: 1, swarmActive: true, cephMounted: true }, + }, + ...overrides, + }; +} + +describe('detectRemoteDrift', () => { + it('reports traefik image changes', () => { + const drifts = detectRemoteDrift(fingerprint(), fingerprint({ traefikImage: 'traefik:v2' }), config); + + expect(drifts.some((drift) => drift.code === 'traefik.image')).toBe(true); + }); + + it('reports traefik service revision changes', () => { + const drifts = detectRemoteDrift(fingerprint(), fingerprint({ traefikServiceVersion: '99' }), config); + + expect(drifts.some((drift) => drift.code === 'traefik.revision')).toBe(true); + }); + + it('reports swarm label changes', () => { + const drifts = detectRemoteDrift( + fingerprint(), + fingerprint({ + swarmNodeLabels: { + 'node-a1': ['site=b'], + 'node-a2': ['site=a'], + }, + }), + config, + ); + + expect(drifts.some((drift) => drift.code === 'swarm.labels')).toBe(true); + }); + + it('reports wireguard deactivation on a managed node', () => { + const drifts = detectRemoteDrift( + fingerprint(), + fingerprint({ + nodes: { + 'node-a1': { wireguardActive: false, wireguardPeerCount: 0, swarmActive: true, cephMounted: true }, + 'node-a2': { wireguardActive: true, wireguardPeerCount: 1, swarmActive: true, cephMounted: true }, + }, + }), + config, + ); + + expect(drifts.some((drift) => drift.code === 'node.node-a1.wireguard')).toBe(true); + }); + + it('reports unexpected wireguard peer counts', () => { + const drifts = detectRemoteDrift( + fingerprint(), + fingerprint({ + nodes: { + 'node-a1': { wireguardActive: true, wireguardPeerCount: 0, swarmActive: true, cephMounted: true }, + 'node-a2': { wireguardActive: true, wireguardPeerCount: 1, swarmActive: true, cephMounted: true }, + }, + }), + config, + ); + + expect(drifts.some((drift) => drift.code === 'node.node-a1.wireguard-peer-count')).toBe(true); + }); + + it('reports per-address VIP holder changes', () => { + const drifts = detectRemoteDrift( + fingerprint({ + vipHolders: { '203.0.113.100': 'node-a1', '203.0.113.101': 'node-a1' }, + }), + fingerprint({ + vipHolders: { '203.0.113.100': 'node-a1', '203.0.113.101': 'node-a2' }, + }), + { + ...config, + vip: { address: '203.0.113.100', interface: 'eth0', backend: 'keepalived', pools: [] }, + }, + ); + + expect(drifts.some((drift) => drift.code === 'vip.holder.203.0.113.101')).toBe(true); + }); +}); diff --git a/libs/domains/loadweaver/shared/feature-cli-cluster/src/lib/drift-detector.ts b/libs/domains/loadweaver/shared/feature-cli-cluster/src/lib/drift-detector.ts new file mode 100644 index 000000000..cde685f73 --- /dev/null +++ b/libs/domains/loadweaver/shared/feature-cli-cluster/src/lib/drift-detector.ts @@ -0,0 +1,201 @@ +import type { DriftFinding } from '@forepath/loadweaver/shared/util-cli-core'; +import type { LoadweaverConfig } from '@forepath/loadweaver/shared/util-cli-core'; + +import type { RemoteFingerprint } from './remote-fingerprint'; +import { deriveExpectedSwarmLabels, missingExpectedSwarmLabels } from '@forepath/loadweaver/shared/util-cli-core'; + +function expectedHostnames(config: LoadweaverConfig): string[] { + return Object.values(config.nodes) + .map((node) => node.hostname) + .sort(); +} + +function expectedWireguardPeerCount(config: LoadweaverConfig): number { + return Math.max(Object.keys(config.nodes).length - 1, 0); +} + +function labelsSignature(labels: Record): string { + return Object.entries(labels) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([nodeId, pairs]) => `${nodeId}:${pairs.join(',')}`) + .join('|'); +} + +export function detectRemoteDrift( + baseline: RemoteFingerprint, + current: RemoteFingerprint, + config: LoadweaverConfig, +): DriftFinding[] { + const drifts: DriftFinding[] = []; + const expectedNodes = Object.keys(config.nodes).sort(); + const expectedPeers = expectedWireguardPeerCount(config); + + if (baseline.swarmNodeHostnames.join(',') !== current.swarmNodeHostnames.join(',')) { + drifts.push({ + code: 'swarm.membership', + message: `Swarm membership changed (${baseline.swarmNodeHostnames.length} -> ${current.swarmNodeHostnames.length} nodes): was [${baseline.swarmNodeHostnames.join(', ')}], now [${current.swarmNodeHostnames.join(', ')}]`, + }); + } + + const expected = expectedHostnames(config); + const unexpectedHostnames = current.swarmNodeHostnames.filter((hostname) => !expected.includes(hostname)); + + if (unexpectedHostnames.length > 0) { + drifts.push({ + code: 'swarm.unknown-hosts', + message: `Swarm contains host(s) not present in loadweaver.yml: ${unexpectedHostnames.join(', ')}`, + }); + } + + const baselineLabels = labelsSignature(baseline.swarmNodeLabels ?? {}); + const currentLabels = labelsSignature(current.swarmNodeLabels ?? {}); + + if (baselineLabels !== currentLabels) { + drifts.push({ + code: 'swarm.labels', + message: 'Swarm node labels changed since last converge', + }); + } + + for (const finding of missingExpectedSwarmLabels(deriveExpectedSwarmLabels(config), current.swarmNodeLabels ?? {})) { + drifts.push({ + code: `swarm.labels.missing.${finding.nodeId}`, + message: `Node ${finding.nodeId} is missing expected Swarm labels: ${finding.missing.join(', ')}`, + }); + } + + if (baseline.cephHealth && current.cephHealth && baseline.cephHealth !== current.cephHealth) { + drifts.push({ + code: 'ceph.health', + message: `Ceph health changed (${baseline.cephHealth} -> ${current.cephHealth})`, + }); + } + + if (baseline.cephMonitorCount !== current.cephMonitorCount) { + drifts.push({ + code: 'ceph.monitors', + message: `Ceph monitor count changed (${baseline.cephMonitorCount} -> ${current.cephMonitorCount})`, + }); + } + + if (Boolean(config.vip) && baseline.vipHolderNodeId !== current.vipHolderNodeId) { + drifts.push({ + code: 'vip.holder', + message: `VIP holder changed (${baseline.vipHolderNodeId ?? 'none'} -> ${current.vipHolderNodeId ?? 'none'})`, + }); + } + + const baselineHolders = baseline.vipHolders ?? {}; + const currentHolders = current.vipHolders ?? {}; + const holderAddresses = [...new Set([...Object.keys(baselineHolders), ...Object.keys(currentHolders)])].sort(); + + for (const address of holderAddresses) { + if ((baselineHolders[address] ?? null) !== (currentHolders[address] ?? null)) { + drifts.push({ + code: `vip.holder.${address}`, + message: `VIP ${address} holder changed (${baselineHolders[address] ?? 'none'} -> ${currentHolders[address] ?? 'none'})`, + }); + } + } + + if (baseline.traefikDeployed !== current.traefikDeployed) { + drifts.push({ + code: 'traefik.presence', + message: `Traefik stack presence changed (deployed=${baseline.traefikDeployed} -> ${current.traefikDeployed})`, + }); + } + + if (baseline.traefikImage && current.traefikImage && baseline.traefikImage !== current.traefikImage) { + drifts.push({ + code: 'traefik.image', + message: `Traefik image changed (${baseline.traefikImage} -> ${current.traefikImage})`, + }); + } + + if ( + baseline.traefikDeployed && + current.traefikDeployed && + baseline.traefikServiceVersion && + current.traefikServiceVersion && + baseline.traefikServiceVersion !== current.traefikServiceVersion + ) { + drifts.push({ + code: 'traefik.revision', + message: `Traefik service revision changed (${baseline.traefikServiceVersion} -> ${current.traefikServiceVersion})`, + }); + } + + if (baseline.traefikReplicas && current.traefikReplicas && baseline.traefikReplicas !== current.traefikReplicas) { + drifts.push({ + code: 'traefik.replicas', + message: `Traefik replica summary changed (${baseline.traefikReplicas} -> ${current.traefikReplicas})`, + }); + } + + if (Boolean(config.vip) && baseline.vipKeepalivedActive !== current.vipKeepalivedActive) { + drifts.push({ + code: 'vip.keepalived', + message: `keepalived active state changed (${baseline.vipKeepalivedActive} -> ${current.vipKeepalivedActive})`, + }); + } + + for (const nodeId of expectedNodes) { + const before = baseline.nodes[nodeId]; + const after = current.nodes[nodeId]; + + if (!before || !after) { + continue; + } + + if (before.wireguardActive && !after.wireguardActive) { + drifts.push({ + code: `node.${nodeId}.wireguard`, + message: `WireGuard on ${nodeId} was active at last converge but is inactive now`, + }); + } + + if (after.wireguardActive && after.wireguardPeerCount !== expectedPeers) { + drifts.push({ + code: `node.${nodeId}.wireguard-peer-count`, + message: `WireGuard on ${nodeId} has ${after.wireguardPeerCount} peer(s); expected ${expectedPeers}`, + }); + } + + if (before.wireguardActive && after.wireguardActive && before.wireguardPeerCount !== after.wireguardPeerCount) { + drifts.push({ + code: `node.${nodeId}.wireguard-peer-change`, + message: `WireGuard peer count on ${nodeId} changed (${before.wireguardPeerCount} -> ${after.wireguardPeerCount})`, + }); + } + + if (before.swarmActive && !after.swarmActive) { + drifts.push({ + code: `node.${nodeId}.swarm`, + message: `Swarm on ${nodeId} was active at last converge but is inactive now`, + }); + } + + if (before.cephMounted && !after.cephMounted) { + drifts.push({ + code: `node.${nodeId}.cephfs`, + message: `CephFS on ${nodeId} was mounted at last converge but is unmounted now`, + }); + } + + if (!before.wireguardActive && after.wireguardActive) { + drifts.push({ + code: `node.${nodeId}.wireguard-unmanaged`, + message: `WireGuard on ${nodeId} is active but was inactive at last converge (possible manual change)`, + }); + } + + if (!before.cephMounted && after.cephMounted) { + drifts.push({ + code: `node.${nodeId}.cephfs-unmanaged`, + message: `CephFS on ${nodeId} is mounted but was unmounted at last converge (possible manual change)`, + }); + } + } + + return drifts; +} diff --git a/libs/domains/loadweaver/shared/feature-cli-cluster/src/lib/mutation-guard.ts b/libs/domains/loadweaver/shared/feature-cli-cluster/src/lib/mutation-guard.ts new file mode 100644 index 000000000..d99010761 --- /dev/null +++ b/libs/domains/loadweaver/shared/feature-cli-cluster/src/lib/mutation-guard.ts @@ -0,0 +1,33 @@ +import type { DriftFinding, LoadweaverContext } from '@forepath/loadweaver/shared/util-cli-core'; + +import { detectRemoteDrift } from './drift-detector'; +import { loadClusterState } from './cluster-state'; +import { collectRemoteFingerprint } from './remote-fingerprint'; +import { detectInventoryDrift, readHostInventories } from './node-inventory.service'; + +export async function inspectRemoteDrift(ctx: LoadweaverContext): Promise { + if (!ctx.config) { + throw new Error('Configuration not loaded'); + } + + const stored = loadClusterState(ctx.options.configPath); + + if (!stored) { + return []; + } + + if (ctx.options.dryRun) { + ctx.logger.warn('Skipping live drift collection in dry-run mode'); + return []; + } + + const current = await collectRemoteFingerprint(ctx); + const hostInventories = await readHostInventories(ctx); + const inventoryNodeIds = Object.keys(ctx.config.nodes); + const fingerprintDrifts = stored?.remoteFingerprint + ? detectRemoteDrift(stored.remoteFingerprint, current, ctx.config) + : []; + const inventoryDrifts = detectInventoryDrift(stored, hostInventories, current, inventoryNodeIds); + + return [...fingerprintDrifts, ...inventoryDrifts]; +} diff --git a/libs/domains/loadweaver/shared/feature-cli-cluster/src/lib/node-inventory.service.spec.ts b/libs/domains/loadweaver/shared/feature-cli-cluster/src/lib/node-inventory.service.spec.ts new file mode 100644 index 000000000..ee2218e3e --- /dev/null +++ b/libs/domains/loadweaver/shared/feature-cli-cluster/src/lib/node-inventory.service.spec.ts @@ -0,0 +1,153 @@ +import type { LoadweaverConfig } from '@forepath/loadweaver/shared/util-cli-core'; + +import type { ClusterState } from './cluster-state'; +import { buildNodeInventory, detectInventoryDrift, type NodeInventoryRecord } from './node-inventory.service'; +import type { RemoteFingerprint } from './remote-fingerprint'; + +const config: LoadweaverConfig = { + version: 1, + cluster: { name: 'prod', primaryManager: 'node-a1' }, + nodes: { + 'node-a1': { hostname: 'a1', wireguardIp: '10.200.0.1', roles: ['manager'] }, + }, + wireguard: { + interface: 'wg0', + port: 51820, + mtu: 1420, + keyRotation: { enabled: false, intervalDays: 90, warnBeforeDays: 14 }, + }, + swarm: { advertiseInterface: 'wg0', overlayNetworks: ['traefik-public'] }, + ceph: { fsName: 'fs', mountPath: '/mnt/cephfs', replication: 3, release: 'quincy' }, + traefik: { image: 'traefik:v3', network: 'traefik-public', mode: 'global' }, + host: { configureFirewall: true }, + volumes: [], +}; + +const fingerprint: RemoteFingerprint = { + capturedAt: '2026-01-01T00:00:00.000Z', + swarmNodeHostnames: ['a1'], + swarmNodeLabels: {}, + traefikImage: null, + traefikDeployed: false, + traefikServiceVersion: null, + traefikReplicas: null, + vipKeepalivedActive: false, + vipHolderNodeId: null, + vipHolders: {}, + cephHealth: null, + cephMonitorCount: 0, + nodes: { + 'node-a1': { wireguardActive: true, wireguardPeerCount: 0, swarmActive: true, cephMounted: true }, + }, +}; + +function inventory(overrides: Partial = {}): NodeInventoryRecord { + return { + inventorySerial: 3, + clusterName: 'prod', + nodeId: 'node-a1', + updatedAt: '2026-01-01T00:00:00.000Z', + lastOperation: 'cluster.update', + node: { hostname: 'a1', roles: ['manager'], wireguardIp: '10.200.0.1' }, + fingerprint: { wireguardActive: true, wireguardPeerCount: 0, swarmActive: true, cephMounted: true }, + ...overrides, + }; +} + +describe('detectInventoryDrift', () => { + const stored: ClusterState = { + version: 1, + clusterName: 'prod', + nodes: ['node-a1'], + swarmLabels: {}, + traefikImage: 'traefik:v3', + traefikMode: 'global', + traefikAcmeEnabled: false, + traefikAcmeChallengeType: null, + traefikAcmeDnsProvider: null, + osdDevices: {}, + cephOsdNodes: [], + overlayNetworks: ['traefik-public'], + volumes: [], + vipConfigured: false, + routingEnabled: false, + routingHubNodes: [], + routingLocalAsn: null, + routingClusterCidr: null, + routingExportWireguardSubnet: true, + routingPeers: [], + nodeHostnames: { 'node-a1': 'a1' }, + updatedAt: '2026-01-01T00:00:00.000Z', + inventorySerial: 3, + desired: config, + remoteFingerprint: fingerprint, + }; + + it('reports missing host inventory files', () => { + const drifts = detectInventoryDrift(stored, { 'node-a1': null }, fingerprint, ['node-a1']); + + expect(drifts.some((drift) => drift.code === 'inventory.missing.node-a1')).toBe(true); + }); + + it('reports serial mismatches against local inventory', () => { + const drifts = detectInventoryDrift(stored, { 'node-a1': inventory({ inventorySerial: 1 }) }, fingerprint, [ + 'node-a1', + ]); + + expect(drifts.some((drift) => drift.code === 'inventory.serial.node-a1')).toBe(true); + }); + + it('reports live probe mismatches against host inventory', () => { + const live: RemoteFingerprint = { + ...fingerprint, + nodes: { + 'node-a1': { wireguardActive: false, wireguardPeerCount: 0, swarmActive: true, cephMounted: true }, + }, + }; + const drifts = detectInventoryDrift(stored, { 'node-a1': inventory() }, live, ['node-a1']); + + expect(drifts.some((drift) => drift.code === 'inventory.live.node-a1.wireguard')).toBe(true); + }); +}); + +describe('buildNodeInventory', () => { + it('builds a per-node inventory record from cluster state', () => { + const state: ClusterState = { + version: 1, + clusterName: 'prod', + nodes: ['node-a1'], + swarmLabels: {}, + traefikImage: 'traefik:v3', + traefikMode: 'global', + traefikAcmeEnabled: false, + traefikAcmeChallengeType: null, + traefikAcmeDnsProvider: null, + osdDevices: {}, + cephOsdNodes: [], + overlayNetworks: ['traefik-public'], + volumes: [], + vipConfigured: false, + routingEnabled: false, + routingHubNodes: [], + routingLocalAsn: null, + routingClusterCidr: null, + routingExportWireguardSubnet: true, + routingPeers: [], + nodeHostnames: { 'node-a1': 'a1' }, + updatedAt: '2026-01-02T00:00:00.000Z', + inventorySerial: 4, + desired: config, + remoteFingerprint: fingerprint, + }; + + expect(buildNodeInventory(state, 'node-a1', 'wireguard.init')).toEqual({ + inventorySerial: 4, + clusterName: 'prod', + nodeId: 'node-a1', + updatedAt: '2026-01-02T00:00:00.000Z', + lastOperation: 'wireguard.init', + node: { hostname: 'a1', roles: ['manager'], wireguardIp: '10.200.0.1' }, + fingerprint: fingerprint.nodes['node-a1'], + }); + }); +}); diff --git a/libs/domains/loadweaver/shared/feature-cli-cluster/src/lib/node-inventory.service.ts b/libs/domains/loadweaver/shared/feature-cli-cluster/src/lib/node-inventory.service.ts new file mode 100644 index 000000000..1f18e6e16 --- /dev/null +++ b/libs/domains/loadweaver/shared/feature-cli-cluster/src/lib/node-inventory.service.ts @@ -0,0 +1,377 @@ +import * as os from 'node:os'; + +import type { DriftFinding, LoadweaverContext } from '@forepath/loadweaver/shared/util-cli-core'; + +import { deriveStateFromConfig, loadClusterState, saveClusterState, type ClusterState } from './cluster-state'; +import type { NodeRemoteSnapshot, RemoteFingerprint } from './remote-fingerprint'; +import { collectRemoteFingerprint } from './remote-fingerprint'; +import { sanitizeDesiredConfig } from './sanitize-desired-config'; + +export const HOST_LOADWEAVER_DIR = '/etc/loadweaver'; +export const HOST_INVENTORY_PATH = `${HOST_LOADWEAVER_DIR}/inventory.json`; +export const HOST_LOCK_PATH = `${HOST_LOADWEAVER_DIR}/lock.json`; + +const HOST_LOCK_STALE_MS = 2 * 60 * 60 * 1000; + +export interface NodeInventoryRecord { + inventorySerial: number; + clusterName: string; + nodeId: string; + updatedAt: string; + lastOperation: string; + node: { + hostname: string; + roles: string[]; + wireguardIp: string; + }; + fingerprint: NodeRemoteSnapshot; +} + +export interface HostLockRecord { + operation: string; + startedAt: string; + operatorHostname: string; +} + +function inventoryNodeIds(ctx: LoadweaverContext): string[] { + const config = ctx.config; + + if (!config) { + return []; + } + + if (ctx.options.local) { + return [config.cluster.primaryManager]; + } + + return Object.keys(config.nodes); +} + +function leaveNodeIdFromOperation(operation: string): string | undefined { + const prefix = 'node.leave.'; + return operation.startsWith(prefix) ? operation.slice(prefix.length) : undefined; +} + +function shellJson(value: unknown): string { + return JSON.stringify(JSON.stringify(value)); +} + +function parseJson(raw: string): T | undefined { + const trimmed = raw.trim(); + + if (!trimmed) { + return undefined; + } + + try { + return JSON.parse(trimmed) as T; + } catch { + return undefined; + } +} + +function isHostLockStale(lock: HostLockRecord): boolean { + const startedAt = Date.parse(lock.startedAt); + + if (Number.isNaN(startedAt)) { + return true; + } + + return Date.now() - startedAt > HOST_LOCK_STALE_MS; +} + +export function buildNodeInventory( + state: ClusterState, + nodeId: string, + operation: string, +): NodeInventoryRecord | undefined { + const node = state.desired?.nodes[nodeId]; + const fingerprint = state.remoteFingerprint?.nodes[nodeId]; + + if (!node || !fingerprint || state.inventorySerial === undefined) { + return undefined; + } + + return { + inventorySerial: state.inventorySerial, + clusterName: state.clusterName, + nodeId, + updatedAt: state.updatedAt, + lastOperation: operation, + node: { + hostname: node.hostname, + roles: [...node.roles], + wireguardIp: node.wireguardIp, + }, + fingerprint, + }; +} + +export async function readHostInventory( + ctx: LoadweaverContext, + nodeId: string, +): Promise { + const result = await ctx + .sshForNode(nodeId) + .execRemote(`cat ${HOST_INVENTORY_PATH} 2>/dev/null || true`, { dryRun: ctx.options.dryRun }); + + return parseJson(result.stdout); +} + +export async function readHostInventories(ctx: LoadweaverContext): Promise> { + const inventories: Record = {}; + + for (const nodeId of inventoryNodeIds(ctx)) { + try { + inventories[nodeId] = (await readHostInventory(ctx, nodeId)) ?? null; + } catch (error: unknown) { + const message = error instanceof Error ? error.message : String(error); + ctx.logger.warn(`Unable to read host inventory on ${nodeId}: ${message}`); + inventories[nodeId] = null; + } + } + + return inventories; +} + +export async function writeHostInventory( + ctx: LoadweaverContext, + nodeId: string, + inventory: NodeInventoryRecord, +): Promise { + await ctx + .sshForNode(nodeId) + .execRemote( + `mkdir -p ${HOST_LOADWEAVER_DIR} && printf '%s\\n' ${shellJson(inventory)} > ${HOST_INVENTORY_PATH} && chmod 0640 ${HOST_INVENTORY_PATH}`, + { dryRun: ctx.options.dryRun }, + ); +} + +export async function removeHostInventory(ctx: LoadweaverContext, nodeId: string): Promise { + await ctx + .sshForNode(nodeId) + .execRemote(`rm -f ${HOST_INVENTORY_PATH} ${HOST_LOCK_PATH}`, { dryRun: ctx.options.dryRun }); +} + +export async function removeHostInventories(ctx: LoadweaverContext, nodeIds?: string[]): Promise { + const targets = nodeIds ?? inventoryNodeIds(ctx); + + for (const nodeId of targets) { + try { + await removeHostInventory(ctx, nodeId); + } catch (error: unknown) { + const message = error instanceof Error ? error.message : String(error); + ctx.logger.warn(`Unable to remove host inventory on ${nodeId}: ${message}`); + } + } +} + +export async function persistClusterInventory(ctx: LoadweaverContext, operation: string): Promise { + if (!ctx.config || ctx.options.dryRun || operation === 'cluster.destroy') { + return; + } + + const previous = loadClusterState(ctx.options.configPath); + const remoteFingerprint = await collectRemoteFingerprint(ctx); + const state = deriveStateFromConfig(ctx.config, remoteFingerprint, { + previous, + bumpSerial: true, + keepDesired: false, + }); + + saveClusterState(ctx.options.configPath, state); + await pushHostInventories(ctx, state, operation); +} + +export async function refreshInventoryFromLive(ctx: LoadweaverContext): Promise { + if (!ctx.config || ctx.options.dryRun) { + return; + } + + const previous = loadClusterState(ctx.options.configPath); + + if (!previous) { + return; + } + + const remoteFingerprint = await collectRemoteFingerprint(ctx); + const state: ClusterState = { + ...previous, + inventorySerial: (previous.inventorySerial ?? 0) + 1, + updatedAt: new Date().toISOString(), + desired: previous.desired ?? sanitizeDesiredConfig(ctx.config), + remoteFingerprint, + }; + + saveClusterState(ctx.options.configPath, state); + await pushHostInventories(ctx, state, 'inventory.refresh'); +} + +async function pushHostInventories(ctx: LoadweaverContext, state: ClusterState, operation: string): Promise { + const skipNodeId = leaveNodeIdFromOperation(operation); + + for (const nodeId of inventoryNodeIds(ctx)) { + if (nodeId === skipNodeId) { + continue; + } + + const inventory = buildNodeInventory(state, nodeId, operation); + + if (!inventory) { + continue; + } + + try { + await writeHostInventory(ctx, nodeId, inventory); + } catch (error: unknown) { + const message = error instanceof Error ? error.message : String(error); + ctx.logger.warn(`Unable to write host inventory on ${nodeId}: ${message}`); + } + } +} + +export async function acquireHostLocks(ctx: LoadweaverContext, operation: string): Promise { + if (!ctx.config || ctx.options.dryRun) { + return []; + } + + const acquired: string[] = []; + const record: HostLockRecord = { + operation, + startedAt: new Date().toISOString(), + operatorHostname: os.hostname(), + }; + + for (const nodeId of inventoryNodeIds(ctx)) { + try { + const existingRaw = await ctx + .sshForNode(nodeId) + .execRemote(`cat ${HOST_LOCK_PATH} 2>/dev/null || true`, { dryRun: false }); + const existing = parseJson(existingRaw.stdout); + + if (existing && !isHostLockStale(existing)) { + throw new Error( + `Host lock held on ${nodeId} by ${existing.operation} (operator ${existing.operatorHostname}, started ${existing.startedAt})`, + ); + } + + if (existing) { + await ctx.sshForNode(nodeId).execRemote(`rm -f ${HOST_LOCK_PATH}`, { dryRun: false }); + } + + const created = await ctx + .sshForNode(nodeId) + .execRemote( + `mkdir -p ${HOST_LOADWEAVER_DIR} && set -C && printf '%s\\n' ${shellJson(record)} > ${HOST_LOCK_PATH}`, + { dryRun: false }, + ); + + if (created.exitCode !== 0) { + throw new Error(`Failed to acquire host lock on ${nodeId}`); + } + + acquired.push(nodeId); + } catch (error: unknown) { + const message = error instanceof Error ? error.message : String(error); + + if (message.includes('Host lock held')) { + await releaseHostLocks(ctx, acquired); + throw error; + } + + ctx.logger.warn(`Unable to acquire host lock on ${nodeId}: ${message}`); + } + } + + return acquired; +} + +export async function releaseHostLocks(ctx: LoadweaverContext, nodeIds: string[]): Promise { + for (const nodeId of nodeIds) { + try { + await ctx.sshForNode(nodeId).execRemote(`rm -f ${HOST_LOCK_PATH}`, { dryRun: ctx.options.dryRun }); + } catch (error: unknown) { + const message = error instanceof Error ? error.message : String(error); + ctx.logger.warn(`Unable to release host lock on ${nodeId}: ${message}`); + } + } +} + +export function detectInventoryDrift( + stored: ClusterState | undefined, + hostInventories: Record, + current: RemoteFingerprint, + nodeIds: string[], +): DriftFinding[] { + const drifts: DriftFinding[] = []; + const localSerial = stored?.inventorySerial; + + for (const nodeId of nodeIds) { + const host = hostInventories[nodeId]; + + if (!host) { + drifts.push({ + code: `inventory.missing.${nodeId}`, + message: `Host inventory missing on ${nodeId} (${HOST_INVENTORY_PATH})`, + }); + continue; + } + + if (host.nodeId !== nodeId) { + drifts.push({ + code: `inventory.identity.${nodeId}`, + message: `Host inventory on ${nodeId} has nodeId ${host.nodeId}`, + }); + } + + if (stored?.clusterName && host.clusterName !== stored.clusterName) { + drifts.push({ + code: `inventory.identity.${nodeId}`, + message: `Host inventory on ${nodeId} is for cluster ${host.clusterName}, expected ${stored.clusterName}`, + }); + } + + if (localSerial !== undefined && host.inventorySerial !== localSerial) { + drifts.push({ + code: `inventory.serial.${nodeId}`, + message: `Host inventory serial on ${nodeId} is ${host.inventorySerial}, local inventory serial is ${localSerial}`, + }); + } + + const live = current.nodes[nodeId]; + + if (!live) { + continue; + } + + if (host.fingerprint.wireguardActive !== live.wireguardActive) { + drifts.push({ + code: `inventory.live.${nodeId}.wireguard`, + message: `WireGuard on ${nodeId} does not match host inventory (inventory=${host.fingerprint.wireguardActive}, live=${live.wireguardActive})`, + }); + } + + if (host.fingerprint.wireguardPeerCount !== live.wireguardPeerCount) { + drifts.push({ + code: `inventory.live.${nodeId}.wireguard-peers`, + message: `WireGuard peer count on ${nodeId} does not match host inventory (${host.fingerprint.wireguardPeerCount} -> ${live.wireguardPeerCount})`, + }); + } + + if (host.fingerprint.swarmActive !== live.swarmActive) { + drifts.push({ + code: `inventory.live.${nodeId}.swarm`, + message: `Swarm on ${nodeId} does not match host inventory (inventory=${host.fingerprint.swarmActive}, live=${live.swarmActive})`, + }); + } + + if (host.fingerprint.cephMounted !== live.cephMounted) { + drifts.push({ + code: `inventory.live.${nodeId}.cephfs`, + message: `CephFS on ${nodeId} does not match host inventory (inventory=${host.fingerprint.cephMounted}, live=${live.cephMounted})`, + }); + } + } + + return drifts; +} diff --git a/libs/domains/loadweaver/shared/feature-cli-cluster/src/lib/remote-fingerprint.ts b/libs/domains/loadweaver/shared/feature-cli-cluster/src/lib/remote-fingerprint.ts new file mode 100644 index 000000000..566b1cb7b --- /dev/null +++ b/libs/domains/loadweaver/shared/feature-cli-cluster/src/lib/remote-fingerprint.ts @@ -0,0 +1,236 @@ +import type { LoadweaverContext } from '@forepath/loadweaver/shared/util-cli-core'; +import { listVipAddresses } from '@forepath/loadweaver/shared/util-cli-core'; + +export interface NodeRemoteSnapshot { + wireguardActive: boolean; + wireguardPeerCount: number; + swarmActive: boolean; + cephMounted: boolean; +} + +export interface RemoteFingerprint { + capturedAt: string; + swarmNodeHostnames: string[]; + swarmNodeLabels: Record; + traefikImage: string | null; + traefikDeployed: boolean; + traefikServiceVersion: string | null; + traefikReplicas: string | null; + vipKeepalivedActive: boolean; + vipHolderNodeId: string | null; + vipHolders: Record; + cephHealth: string | null; + cephMonitorCount: number; + nodes: Record; +} + +export async function collectRemoteFingerprint(ctx: LoadweaverContext): Promise { + if (!ctx.config) { + throw new Error('Configuration not loaded'); + } + + const config = ctx.config; + const primary = config.cluster.primaryManager; + const nodes: Record = {}; + + for (const nodeId of Object.keys(config.nodes)) { + nodes[nodeId] = await probeNode(ctx, nodeId); + } + + const swarmNodeHostnames = await probeSwarmNodeHostnames(ctx, primary); + const swarmNodeLabels = await probeSwarmNodeLabels(ctx, primary); + const traefik = await probeTraefik(ctx, primary); + const vipKeepalivedActive = config.vip ? await probeKeepalived(ctx, primary) : false; + const vipHolders = config.vip ? await probeVipHolders(ctx) : {}; + const vipHolderNodeId = config.vip?.address ? (vipHolders[config.vip.address] ?? null) : null; + const ceph = await probeCeph(ctx, primary); + + return { + capturedAt: new Date().toISOString(), + swarmNodeHostnames, + swarmNodeLabels, + traefikImage: traefik.image, + traefikDeployed: traefik.deployed, + traefikServiceVersion: traefik.serviceVersion, + traefikReplicas: traefik.replicas, + vipKeepalivedActive, + vipHolderNodeId, + vipHolders, + cephHealth: ceph.health, + cephMonitorCount: ceph.monitorCount, + nodes, + }; +} + +async function probeNode(ctx: LoadweaverContext, nodeId: string): Promise { + const config = ctx.config!; + const iface = config.wireguard.interface; + const mountPath = config.ceph.mountPath; + + const wireguard = await ctx + .sshForNode(nodeId) + .execRemote(`wg show ${iface} >/dev/null 2>&1 && echo active || echo inactive`, { dryRun: ctx.options.dryRun }); + const peerCount = await ctx + .sshForNode(nodeId) + .execRemote(`wg show ${iface} peers 2>/dev/null | wc -l | tr -d ' '`, { dryRun: ctx.options.dryRun }); + const swarm = await ctx + .sshForNode(nodeId) + .execRemote(`docker info --format '{{.Swarm.LocalNodeState}}' 2>/dev/null || echo inactive`, { + dryRun: ctx.options.dryRun, + }); + const ceph = await ctx + .sshForNode(nodeId) + .execRemote(`mountpoint -q ${mountPath} && echo mounted || echo unmounted`, { dryRun: ctx.options.dryRun }); + + return { + wireguardActive: wireguard.stdout.trim() === 'active', + wireguardPeerCount: Number.parseInt(peerCount.stdout.trim(), 10) || 0, + swarmActive: swarm.stdout.trim() === 'active', + cephMounted: ceph.stdout.trim() === 'mounted', + }; +} + +async function probeSwarmNodeHostnames(ctx: LoadweaverContext, primaryNodeId: string): Promise { + const result = await ctx + .sshForNode(primaryNodeId) + .execRemote(`docker node ls --format '{{.Hostname}}' 2>/dev/null || true`, { dryRun: ctx.options.dryRun }); + + return result.stdout + .split('\n') + .map((line) => line.trim()) + .filter((line) => line.length > 0) + .sort(); +} + +async function probeSwarmNodeLabels(ctx: LoadweaverContext, primaryNodeId: string): Promise> { + const config = ctx.config!; + const hostnameToNodeId = Object.fromEntries( + Object.entries(config.nodes).map(([nodeId, node]) => [node.hostname, nodeId]), + ); + const labels: Record = {}; + + const listing = await ctx + .sshForNode(primaryNodeId) + .execRemote(`docker node ls --format '{{.Hostname}}\t{{.ID}}' 2>/dev/null || true`, { + dryRun: ctx.options.dryRun, + }); + + for (const line of listing.stdout + .split('\n') + .map((entry) => entry.trim()) + .filter(Boolean)) { + const [hostname, dockerNodeId] = line.split('\t'); + const loadweaverNodeId = hostname ? hostnameToNodeId[hostname.trim()] : undefined; + + if (!loadweaverNodeId || !dockerNodeId) { + continue; + } + + const inspect = await ctx + .sshForNode(primaryNodeId) + .execRemote( + `docker node inspect ${dockerNodeId.trim()} --format '{{range $k,$v := .Spec.Labels}}{{$k}}={{$v}}\n{{end}}' 2>/dev/null || true`, + { dryRun: ctx.options.dryRun }, + ); + + labels[loadweaverNodeId] = inspect.stdout + .split('\n') + .map((entry) => entry.trim()) + .filter(Boolean) + .sort(); + } + + return labels; +} + +async function probeTraefik( + ctx: LoadweaverContext, + primaryNodeId: string, +): Promise<{ + deployed: boolean; + image: string | null; + serviceVersion: string | null; + replicas: string | null; +}> { + const imageResult = await ctx + .sshForNode(primaryNodeId) + .execRemote(`docker stack services traefik --format '{{.Image}}' 2>/dev/null | head -n 1`, { + dryRun: ctx.options.dryRun, + }); + const versionResult = await ctx + .sshForNode(primaryNodeId) + .execRemote(`docker service inspect traefik_traefik --format '{{.Version.Index}}' 2>/dev/null || true`, { + dryRun: ctx.options.dryRun, + }); + const replicasResult = await ctx + .sshForNode(primaryNodeId) + .execRemote(`docker stack services traefik --format '{{.Name}}:{{.Replicas}}' 2>/dev/null | head -n 1 || true`, { + dryRun: ctx.options.dryRun, + }); + + const image = imageResult.stdout.trim(); + + if (!image) { + return { deployed: false, image: null, serviceVersion: null, replicas: null }; + } + + return { + deployed: true, + image, + serviceVersion: versionResult.stdout.trim() || null, + replicas: replicasResult.stdout.trim() || null, + }; +} + +async function probeKeepalived(ctx: LoadweaverContext, primaryNodeId: string): Promise { + const result = await ctx + .sshForNode(primaryNodeId) + .execRemote(`systemctl is-active keepalived 2>/dev/null || echo inactive`, { dryRun: ctx.options.dryRun }); + + return result.stdout.trim() === 'active'; +} + +async function probeVipHolders(ctx: LoadweaverContext): Promise> { + const holders: Record = {}; + + for (const address of listVipAddresses(ctx.config!)) { + holders[address] = await probeVipHolder(ctx, address); + } + + return holders; +} + +async function probeVipHolder(ctx: LoadweaverContext, vipAddress: string): Promise { + for (const nodeId of Object.keys(ctx.config!.nodes)) { + const result = await ctx + .sshForNode(nodeId) + .execRemote(`ip -4 addr show | grep -F '${vipAddress}' || true`, { dryRun: ctx.options.dryRun }); + + if (result.stdout.includes(vipAddress)) { + return nodeId; + } + } + + return null; +} + +async function probeCeph( + ctx: LoadweaverContext, + primaryNodeId: string, +): Promise<{ health: string | null; monitorCount: number }> { + const healthResult = await ctx + .sshForNode(primaryNodeId) + .execRemote(`ceph -s 2>/dev/null | awk '/health:/ {print $2; exit}' || true`, { dryRun: ctx.options.dryRun }); + const monResult = await ctx + .sshForNode(primaryNodeId) + .execRemote(`ceph mon stat 2>/dev/null | awk -F'=' '/mons/{print $2; exit}' | tr -d ' ' || true`, { + dryRun: ctx.options.dryRun, + }); + + const monitorCount = Number.parseInt(monResult.stdout.trim(), 10); + + return { + health: healthResult.stdout.trim() || null, + monitorCount: Number.isNaN(monitorCount) ? 0 : monitorCount, + }; +} diff --git a/libs/domains/loadweaver/shared/feature-cli-cluster/src/lib/sanitize-desired-config.spec.ts b/libs/domains/loadweaver/shared/feature-cli-cluster/src/lib/sanitize-desired-config.spec.ts new file mode 100644 index 000000000..4c28e6f7f --- /dev/null +++ b/libs/domains/loadweaver/shared/feature-cli-cluster/src/lib/sanitize-desired-config.spec.ts @@ -0,0 +1,61 @@ +import type { LoadweaverConfig } from '@forepath/loadweaver/shared/util-cli-core'; + +import { AUTH_PASS_REDACTED, sanitizeDesiredConfig } from './sanitize-desired-config'; + +const config: LoadweaverConfig = { + version: 1, + cluster: { name: 'prod', primaryManager: 'node-a1' }, + nodes: { + 'node-a1': { hostname: 'a1', wireguardIp: '10.200.0.1', roles: ['manager'] }, + }, + wireguard: { + interface: 'wg0', + port: 51820, + mtu: 1420, + keyRotation: { enabled: false, intervalDays: 90, warnBeforeDays: 14 }, + }, + swarm: { advertiseInterface: 'wg0', overlayNetworks: ['traefik-public'] }, + ceph: { fsName: 'fs', mountPath: '/mnt/cephfs', replication: 3, release: 'quincy' }, + traefik: { image: 'traefik:v3', network: 'traefik-public', mode: 'global' }, + host: { configureFirewall: true }, + volumes: [], + vip: { + address: '10.200.0.100', + interface: 'eth0', + backend: 'keepalived', + authPass: 'secret01', + pools: [], + }, +}; + +describe('sanitizeDesiredConfig', () => { + it('redacts vip.authPass without mutating the source config', () => { + const sanitized = sanitizeDesiredConfig(config); + + expect(sanitized.vip?.authPass).toBe(AUTH_PASS_REDACTED); + expect(config.vip?.authPass).toBe('secret01'); + }); + + it('redacts pool authPass values', () => { + const withPools: LoadweaverConfig = { + ...config, + vip: { + ...config.vip!, + pools: [ + { + name: 'postgres', + address: '10.200.0.101', + authPass: 'poolpass', + healthCheck: { type: 'tcp', path: '/' }, + listeners: [], + }, + ], + }, + }; + + const sanitized = sanitizeDesiredConfig(withPools); + + expect(sanitized.vip?.pools?.[0].authPass).toBe(AUTH_PASS_REDACTED); + expect(withPools.vip?.pools?.[0].authPass).toBe('poolpass'); + }); +}); diff --git a/libs/domains/loadweaver/shared/feature-cli-cluster/src/lib/sanitize-desired-config.ts b/libs/domains/loadweaver/shared/feature-cli-cluster/src/lib/sanitize-desired-config.ts new file mode 100644 index 000000000..822d58bb9 --- /dev/null +++ b/libs/domains/loadweaver/shared/feature-cli-cluster/src/lib/sanitize-desired-config.ts @@ -0,0 +1,19 @@ +import type { LoadweaverConfig } from '@forepath/loadweaver/shared/util-cli-core'; + +export const AUTH_PASS_REDACTED = '[redacted]'; + +export function sanitizeDesiredConfig(config: LoadweaverConfig): LoadweaverConfig { + const desired = structuredClone(config); + + if (desired.vip?.authPass) { + desired.vip.authPass = AUTH_PASS_REDACTED; + } + + for (const pool of desired.vip?.pools ?? []) { + if (pool.authPass) { + pool.authPass = AUTH_PASS_REDACTED; + } + } + + return desired; +} diff --git a/libs/domains/loadweaver/shared/feature-cli-cluster/tsconfig.json b/libs/domains/loadweaver/shared/feature-cli-cluster/tsconfig.json new file mode 100644 index 000000000..9e5fde1c6 --- /dev/null +++ b/libs/domains/loadweaver/shared/feature-cli-cluster/tsconfig.json @@ -0,0 +1,16 @@ +{ + "extends": "../../../../../tsconfig.base.json", + "compilerOptions": { + "module": "commonjs" + }, + "files": [], + "include": [], + "references": [ + { + "path": "./tsconfig.lib.json" + }, + { + "path": "./tsconfig.spec.json" + } + ] +} diff --git a/libs/domains/loadweaver/shared/feature-cli-cluster/tsconfig.lib.json b/libs/domains/loadweaver/shared/feature-cli-cluster/tsconfig.lib.json new file mode 100644 index 000000000..841e950a0 --- /dev/null +++ b/libs/domains/loadweaver/shared/feature-cli-cluster/tsconfig.lib.json @@ -0,0 +1,15 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "../../../../../dist/out-tsc", + "declaration": true, + "types": ["node"] + }, + "include": ["src/**/*.ts"], + "exclude": [ + "jest.config.ts", + "jest.config.cts", + "src/**/*.spec.ts", + "src/**/*.test.ts" + ] +} diff --git a/libs/domains/loadweaver/shared/feature-cli-cluster/tsconfig.spec.json b/libs/domains/loadweaver/shared/feature-cli-cluster/tsconfig.spec.json new file mode 100644 index 000000000..40cd1d632 --- /dev/null +++ b/libs/domains/loadweaver/shared/feature-cli-cluster/tsconfig.spec.json @@ -0,0 +1,17 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "../../../../../dist/out-tsc", + "module": "commonjs", + "moduleResolution": "node10", + "types": ["jest", "node"] + }, + "include": [ + "jest.config.ts", + "jest.config.cts", + "src/**/*.test.ts", + "src/**/*.spec.ts", + "src/**/*.d.ts", + "src/**/*.tpl" + ] +} diff --git a/libs/domains/loadweaver/shared/feature-cli-diag/.eslintrc.json b/libs/domains/loadweaver/shared/feature-cli-diag/.eslintrc.json new file mode 100644 index 000000000..b33995654 --- /dev/null +++ b/libs/domains/loadweaver/shared/feature-cli-diag/.eslintrc.json @@ -0,0 +1,18 @@ +{ + "extends": ["../../../../../.eslintrc.json"], + "ignorePatterns": ["!**/*"], + "overrides": [ + { + "files": ["*.ts", "*.tsx", "*.js", "*.jsx"], + "rules": {} + }, + { + "files": ["*.ts", "*.tsx"], + "rules": {} + }, + { + "files": ["*.js", "*.jsx"], + "rules": {} + } + ] +} diff --git a/libs/domains/loadweaver/shared/feature-cli-diag/README.md b/libs/domains/loadweaver/shared/feature-cli-diag/README.md new file mode 100644 index 000000000..c7c25eada --- /dev/null +++ b/libs/domains/loadweaver/shared/feature-cli-diag/README.md @@ -0,0 +1,7 @@ +# loadweaver-shared-feature-cli-diag + +This library was generated with [Nx](https://nx.dev). + +## Running unit tests + +Run `nx test loadweaver-shared-feature-cli-diag` to execute the unit tests via [Jest](https://jestjs.io). diff --git a/libs/domains/loadweaver/shared/feature-cli-diag/jest.config.cts b/libs/domains/loadweaver/shared/feature-cli-diag/jest.config.cts new file mode 100644 index 000000000..3573ce0a1 --- /dev/null +++ b/libs/domains/loadweaver/shared/feature-cli-diag/jest.config.cts @@ -0,0 +1,11 @@ +module.exports = { + displayName: 'loadweaver-shared-feature-cli-diag', + preset: '../../../../../jest.preset.cjs', + testEnvironment: 'node', + transform: { + '^.+\\.[tj]s$': ['ts-jest', { tsconfig: '/tsconfig.spec.json' }], + }, + moduleFileExtensions: ['ts', 'js', 'html'], + coverageDirectory: + '../../../../../coverage/libs/domains/loadweaver/shared/feature-cli-diag', +}; diff --git a/libs/domains/loadweaver/shared/feature-cli-diag/project.json b/libs/domains/loadweaver/shared/feature-cli-diag/project.json new file mode 100644 index 000000000..4f44620d9 --- /dev/null +++ b/libs/domains/loadweaver/shared/feature-cli-diag/project.json @@ -0,0 +1,16 @@ +{ + "name": "loadweaver-shared-feature-cli-diag", + "$schema": "../../../../../node_modules/nx/schemas/project-schema.json", + "sourceRoot": "libs/domains/loadweaver/shared/feature-cli-diag/src", + "projectType": "library", + "tags": ["domain:loadweaver", "scope:shared", "type:feature"], + "targets": { + "test": { + "executor": "@nx/jest:jest", + "outputs": ["{workspaceRoot}/coverage/{projectRoot}"], + "options": { + "jestConfig": "libs/domains/loadweaver/shared/feature-cli-diag/jest.config.cts" + } + } + } +} diff --git a/libs/domains/loadweaver/shared/feature-cli-diag/src/index.ts b/libs/domains/loadweaver/shared/feature-cli-diag/src/index.ts new file mode 100644 index 000000000..4f5c74b79 --- /dev/null +++ b/libs/domains/loadweaver/shared/feature-cli-diag/src/index.ts @@ -0,0 +1,2 @@ +export { registerDiagCommands } from './lib/command'; +export { DiagService } from './lib/diag.service'; diff --git a/libs/domains/loadweaver/shared/feature-cli-diag/src/lib/command.ts b/libs/domains/loadweaver/shared/feature-cli-diag/src/lib/command.ts new file mode 100644 index 000000000..b618a4c50 --- /dev/null +++ b/libs/domains/loadweaver/shared/feature-cli-diag/src/lib/command.ts @@ -0,0 +1,36 @@ +import type { Command } from 'commander'; + +import type { LoadweaverContext } from '@forepath/loadweaver/shared/util-cli-core'; +import { withExamples } from '@forepath/loadweaver/shared/util-cli-core'; + +import { DiagService } from './diag.service'; + +export function registerDiagCommands(program: Command, getCtx: (command: Command) => LoadweaverContext): void { + const diag = program.command('diag').description('Diagnostics and health checks'); + withExamples(diag, ['loadweaver diag all', 'loadweaver diag ping', 'loadweaver diag ssh node-a1']); + + const all = diag + .command('all') + .description('Run comprehensive cluster diagnostics') + .action(async function (this: Command) { + await new DiagService(getCtx(this)).runAll(); + }); + withExamples(all, ['loadweaver diag all', 'loadweaver --verbose diag all']); + + const ping = diag + .command('ping') + .description('Ping all nodes over WireGuard') + .action(async function (this: Command) { + await new DiagService(getCtx(this)).ping(); + }); + withExamples(ping, ['loadweaver diag ping']); + + const ssh = diag + .command('ssh') + .description('Verify SSH target resolution and host software readiness for a node') + .argument('', 'Node identifier') + .action(async function (this: Command, nodeId: string) { + await new DiagService(getCtx(this)).ssh(nodeId); + }); + withExamples(ssh, ['loadweaver diag ssh node-a1', 'loadweaver --json diag ssh node-a1']); +} diff --git a/libs/domains/loadweaver/shared/feature-cli-diag/src/lib/diag.service.ts b/libs/domains/loadweaver/shared/feature-cli-diag/src/lib/diag.service.ts new file mode 100644 index 000000000..297787759 --- /dev/null +++ b/libs/domains/loadweaver/shared/feature-cli-diag/src/lib/diag.service.ts @@ -0,0 +1,84 @@ +import type { LoadweaverContext } from '@forepath/loadweaver/shared/util-cli-core'; +import { printStructuredOutput, resolveSshTarget } from '@forepath/loadweaver/shared/util-cli-core'; + +import { HostService } from '@forepath/loadweaver/shared/feature-cli-host'; +import { CephService } from '@forepath/loadweaver/shared/feature-cli-ceph'; +import { SwarmService } from '@forepath/loadweaver/shared/feature-cli-swarm'; +import { TraefikService } from '@forepath/loadweaver/shared/feature-cli-traefik'; +import { VipService } from '@forepath/loadweaver/shared/feature-cli-vip'; +import { VolumeService } from '@forepath/loadweaver/shared/feature-cli-volume'; +import { WireguardService } from '@forepath/loadweaver/shared/feature-cli-wireguard'; + +export class DiagService { + constructor(private readonly ctx: LoadweaverContext) {} + + async runAll(): Promise { + await this.ping(); + await new HostService(this.ctx).status(); + await new WireguardService(this.ctx).status(); + await new SwarmService(this.ctx).status(); + await new CephService(this.ctx).status(); + await new VolumeService(this.ctx).list(); + await new TraefikService(this.ctx).status(); + + if (this.ctx.config?.vip) { + await new VipService(this.ctx).status(); + } + } + + async ping(): Promise { + const config = this.requireConfig(); + const nodes = Object.entries(config.nodes); + const results: Array<{ source: string; target: string; exitCode: number }> = []; + + for (const [sourceId, source] of nodes) { + for (const [targetId, target] of nodes) { + if (sourceId === targetId) { + continue; + } + + const result = await this.ctx + .sshForNode(sourceId) + .execRemote(`ping -c 1 -W 1 ${target.wireguardIp} || true`, { dryRun: this.ctx.options.dryRun }); + + results.push({ source: sourceId, target: targetId, exitCode: result.exitCode }); + + if (!this.ctx.options.json) { + this.ctx.logger.info(`${source.hostname} -> ${targetId}: exit=${result.exitCode}`); + } + } + } + + if (this.ctx.options.json) { + printStructuredOutput(this.ctx, { ping: results }); + } + } + + async ssh(nodeId: string): Promise { + const config = this.requireConfig(); + + if (!config.nodes[nodeId]) { + throw new Error(`Unknown node: ${nodeId}`); + } + + const target = resolveSshTarget(config, nodeId); + const host = new HostService(this.ctx); + const readiness = await host.verifyNode(nodeId); + const payload = { + nodeId, + target, + hostReady: readiness.passed, + message: readiness.message, + }; + + printStructuredOutput(this.ctx, payload); + } + + private requireConfig() { + if (!this.ctx.config) { + throw new Error('Configuration not loaded'); + } + + return this.ctx.config; + } +} diff --git a/libs/domains/loadweaver/shared/feature-cli-diag/tsconfig.json b/libs/domains/loadweaver/shared/feature-cli-diag/tsconfig.json new file mode 100644 index 000000000..9e5fde1c6 --- /dev/null +++ b/libs/domains/loadweaver/shared/feature-cli-diag/tsconfig.json @@ -0,0 +1,16 @@ +{ + "extends": "../../../../../tsconfig.base.json", + "compilerOptions": { + "module": "commonjs" + }, + "files": [], + "include": [], + "references": [ + { + "path": "./tsconfig.lib.json" + }, + { + "path": "./tsconfig.spec.json" + } + ] +} diff --git a/libs/domains/loadweaver/shared/feature-cli-diag/tsconfig.lib.json b/libs/domains/loadweaver/shared/feature-cli-diag/tsconfig.lib.json new file mode 100644 index 000000000..841e950a0 --- /dev/null +++ b/libs/domains/loadweaver/shared/feature-cli-diag/tsconfig.lib.json @@ -0,0 +1,15 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "../../../../../dist/out-tsc", + "declaration": true, + "types": ["node"] + }, + "include": ["src/**/*.ts"], + "exclude": [ + "jest.config.ts", + "jest.config.cts", + "src/**/*.spec.ts", + "src/**/*.test.ts" + ] +} diff --git a/libs/domains/loadweaver/shared/feature-cli-diag/tsconfig.spec.json b/libs/domains/loadweaver/shared/feature-cli-diag/tsconfig.spec.json new file mode 100644 index 000000000..c714b3c64 --- /dev/null +++ b/libs/domains/loadweaver/shared/feature-cli-diag/tsconfig.spec.json @@ -0,0 +1,16 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "../../../../../dist/out-tsc", + "module": "commonjs", + "moduleResolution": "node10", + "types": ["jest", "node"] + }, + "include": [ + "jest.config.ts", + "jest.config.cts", + "src/**/*.test.ts", + "src/**/*.spec.ts", + "src/**/*.d.ts" + ] +} diff --git a/libs/domains/loadweaver/shared/feature-cli-host/jest.config.cts b/libs/domains/loadweaver/shared/feature-cli-host/jest.config.cts new file mode 100644 index 000000000..44224aa0b --- /dev/null +++ b/libs/domains/loadweaver/shared/feature-cli-host/jest.config.cts @@ -0,0 +1,11 @@ +module.exports = { + displayName: 'loadweaver-shared-feature-cli-host', + preset: '../../../../../jest.preset.cjs', + testEnvironment: 'node', + transform: { + '^.+\\.[tj]s$': ['ts-jest', { tsconfig: '/tsconfig.spec.json' }], + }, + moduleFileExtensions: ['ts', 'js', 'html'], + coverageDirectory: + '../../../../../coverage/libs/domains/loadweaver/shared/feature-cli-host', +}; diff --git a/libs/domains/loadweaver/shared/feature-cli-host/project.json b/libs/domains/loadweaver/shared/feature-cli-host/project.json new file mode 100644 index 000000000..020dd1ba7 --- /dev/null +++ b/libs/domains/loadweaver/shared/feature-cli-host/project.json @@ -0,0 +1,16 @@ +{ + "name": "loadweaver-shared-feature-cli-host", + "$schema": "../../../../../node_modules/nx/schemas/project-schema.json", + "sourceRoot": "libs/domains/loadweaver/shared/feature-cli-host/src", + "projectType": "library", + "tags": ["domain:loadweaver", "scope:shared", "type:feature"], + "targets": { + "test": { + "executor": "@nx/jest:jest", + "outputs": ["{workspaceRoot}/coverage/{projectRoot}"], + "options": { + "jestConfig": "libs/domains/loadweaver/shared/feature-cli-host/jest.config.cts" + } + } + } +} diff --git a/libs/domains/loadweaver/shared/feature-cli-host/src/index.ts b/libs/domains/loadweaver/shared/feature-cli-host/src/index.ts new file mode 100644 index 000000000..510586092 --- /dev/null +++ b/libs/domains/loadweaver/shared/feature-cli-host/src/index.ts @@ -0,0 +1,3 @@ +export { registerHostCommands } from './lib/command'; +export { HostService } from './lib/host.service'; +export { buildHostBootstrapScript, buildHostVerificationScript, parseOsRelease } from './lib/host-provision-script'; diff --git a/libs/domains/loadweaver/shared/feature-cli-host/src/lib/command.ts b/libs/domains/loadweaver/shared/feature-cli-host/src/lib/command.ts new file mode 100644 index 000000000..b9fb919b9 --- /dev/null +++ b/libs/domains/loadweaver/shared/feature-cli-host/src/lib/command.ts @@ -0,0 +1,46 @@ +import type { Command } from 'commander'; + +import type { LoadweaverContext } from '@forepath/loadweaver/shared/util-cli-core'; +import { runGuardedMutation, withExamples } from '@forepath/loadweaver/shared/util-cli-core'; + +import { HostService } from './host.service'; + +export function registerHostCommands(program: Command, getCtx: (command: Command) => LoadweaverContext): void { + const host = program.command('host').description('Host OS provisioning and verification'); + withExamples(host, ['loadweaver host bootstrap', 'loadweaver --dry-run host verify']); + + const bootstrap = host + .command('bootstrap') + .description('Install Docker, WireGuard, keepalived, and cephadm on cluster nodes') + .argument('[nodeId]', 'Optional node id (defaults to all nodes)') + .action(async function (this: Command, nodeId?: string) { + const ctx = getCtx(this); + const operation = nodeId ? `host.bootstrap.${nodeId}` : 'host.bootstrap'; + await runGuardedMutation(ctx, operation, async () => { + const service = new HostService(ctx); + + if (nodeId) { + await service.bootstrapNode(nodeId); + } else { + await service.bootstrapAll(); + } + }); + }); + withExamples(bootstrap, ['loadweaver host bootstrap', 'loadweaver host bootstrap node-a1']); + + const verify = host + .command('verify') + .description('Verify required host packages are installed on cluster nodes') + .action(async function (this: Command) { + await new HostService(getCtx(this)).verifyAll(); + }); + withExamples(verify, ['loadweaver host verify']); + + const status = host + .command('status') + .description('Show host software readiness per node') + .action(async function (this: Command) { + await new HostService(getCtx(this)).status(); + }); + withExamples(status, ['loadweaver host status']); +} diff --git a/libs/domains/loadweaver/shared/feature-cli-host/src/lib/host-provision-script.spec.ts b/libs/domains/loadweaver/shared/feature-cli-host/src/lib/host-provision-script.spec.ts new file mode 100644 index 000000000..5fa97c615 --- /dev/null +++ b/libs/domains/loadweaver/shared/feature-cli-host/src/lib/host-provision-script.spec.ts @@ -0,0 +1,119 @@ +import { buildHostBootstrapScript, buildHostVerificationScript, parseOsRelease } from './host-provision-script'; + +describe('host provision scripts', () => { + it('parses os-release content', () => { + expect( + parseOsRelease(`ID=debian +VERSION_CODENAME=bookworm +`), + ).toEqual({ id: 'debian', versionCodename: 'bookworm' }); + }); + + it('builds a docker and wireguard bootstrap script for debian', () => { + const script = buildHostBootstrapScript( + { id: 'debian', versionCodename: 'bookworm' }, + { + installKeepalived: true, + installHaproxy: true, + installBird: false, + installCephadm: true, + cephRelease: 'quincy', + wireguardPort: 51820, + configureFirewall: true, + listenerPorts: [5432], + }, + ); + + expect(script).toContain('docker-ce'); + expect(script).toContain('wireguard-tools'); + expect(script).toContain('keepalived'); + expect(script).toContain('haproxy'); + expect(script).toContain('cephadm'); + expect(script).toContain('ufw allow 2377/tcp'); + expect(script).toContain("ufw allow 5432/tcp comment 'loadweaver-vip-pool'"); + }); + + it('omits optional packages when not requested', () => { + const script = buildHostBootstrapScript( + { id: 'ubuntu', versionCodename: 'jammy' }, + { + installKeepalived: false, + installHaproxy: false, + installBird: false, + installCephadm: false, + cephRelease: 'quincy', + wireguardPort: 51820, + configureFirewall: false, + }, + ); + + expect(script).not.toContain('keepalived'); + expect(script).not.toContain('haproxy'); + expect(script).not.toContain('cephadm add-repo'); + }); + + it('builds verification commands for required software', () => { + const script = buildHostVerificationScript({ + docker: true, + wireguard: true, + keepalived: true, + haproxy: true, + bird: false, + cephadm: false, + }); + + expect(script).toContain('command -v docker'); + expect(script).toContain('command -v keepalived'); + expect(script).toContain('command -v haproxy'); + expect(script).not.toContain('cephadm'); + }); +}); + +describe('HostService', () => { + it('bootstraps all nodes in dry-run mode', async () => { + const commands: string[] = []; + const ctx = { + options: { configPath: './loadweaver.yml', dryRun: true, verbose: false, debug: false, yes: false, local: false }, + config: { + version: 1, + cluster: { name: 'test', primaryManager: 'node-a1' }, + nodes: { + 'node-a1': { hostname: 'a1', wireguardIp: '10.200.0.1', roles: ['manager', 'ceph-mon'] }, + }, + wireguard: { + interface: 'wg0', + port: 51820, + mtu: 1420, + keyRotation: { enabled: false, intervalDays: 90, warnBeforeDays: 14 }, + }, + swarm: { advertiseInterface: 'wg0', overlayNetworks: [] }, + ceph: { fsName: 'fs', mountPath: '/mnt/cephfs', replication: 3, release: 'quincy' }, + traefik: { image: 'traefik:v3', network: 'traefik-public', mode: 'global' as const }, + volumes: [], + host: { configureFirewall: true }, + vip: { address: '10.0.0.1', interface: 'eth0', backend: 'keepalived' as const }, + }, + logger: { error: () => undefined, warn: () => undefined, info: () => undefined, debug: () => undefined }, + sshForNode: () => ({ + execRemote: async (command: string) => { + commands.push(command); + return { + command, + stdout: command.includes('os-release') ? 'ID=debian\nVERSION_CODENAME=bookworm\n' : 'ok', + stderr: '', + exitCode: 0, + dryRun: true, + }; + }, + uploadFile: async () => ({ command: '', stdout: '', stderr: '', exitCode: 0, dryRun: true }), + }), + sshTargetForNode: () => ({ host: '127.0.0.1', user: 'root' }), + }; + + const { HostService } = await import('./host.service'); + await new HostService(ctx as never).bootstrapAll(); + + expect(commands.some((command) => command.includes('docker-ce'))).toBe(true); + expect(commands.some((command) => command.includes('wireguard-tools'))).toBe(true); + }); +}); diff --git a/libs/domains/loadweaver/shared/feature-cli-host/src/lib/host-provision-script.ts b/libs/domains/loadweaver/shared/feature-cli-host/src/lib/host-provision-script.ts new file mode 100644 index 000000000..8e7772780 --- /dev/null +++ b/libs/domains/loadweaver/shared/feature-cli-host/src/lib/host-provision-script.ts @@ -0,0 +1,197 @@ +export interface HostProvisionOptions { + installKeepalived: boolean; + installHaproxy: boolean; + installBird: boolean; + installCephadm: boolean; + cephRelease: string; + wireguardPort: number; + configureFirewall: boolean; + aptProxy?: string; + listenerPorts?: number[]; +} + +export interface HostOsInfo { + id: string; + versionCodename: string; +} + +export function parseOsRelease(content: string): HostOsInfo { + const values = Object.fromEntries( + content + .split('\n') + .map((line) => line.trim()) + .filter((line) => line.includes('=')) + .map((line) => { + const index = line.indexOf('='); + const key = line.slice(0, index); + const rawValue = line.slice(index + 1).replace(/^"|"$/g, ''); + + return [key, rawValue]; + }), + ); + + const id = values['ID']; + + if (!id) { + throw new Error('Unable to detect OS ID from /etc/os-release'); + } + + return { + id, + versionCodename: values['VERSION_CODENAME'] ?? values['VERSION_ID'] ?? 'stable', + }; +} + +function aptProxyBlock(aptProxy?: string): string { + if (!aptProxy) { + return ''; + } + + return ` +export http_proxy='${aptProxy.replace(/'/g, `'\\''`)}' +export https_proxy='${aptProxy.replace(/'/g, `'\\''`)}' +`; +} + +function firewallBlock(configureFirewall: boolean, wireguardPort: number, listenerPorts: number[] = []): string { + if (!configureFirewall) { + return ''; + } + + const listenerRules = listenerPorts + .map((port) => ` ufw allow ${port}/tcp comment 'loadweaver-vip-pool' || true`) + .join('\n'); + + return ` +if command -v ufw >/dev/null 2>&1; then + ufw allow 2377/tcp comment 'loadweaver-swarm' || true + ufw allow 7946/tcp comment 'loadweaver-swarm' || true + ufw allow 7946/udp comment 'loadweaver-swarm' || true + ufw allow 4789/udp comment 'loadweaver-swarm' || true + ufw allow ${wireguardPort}/udp comment 'loadweaver-wireguard' || true + ufw allow 80/tcp comment 'loadweaver-traefik' || true + ufw allow 443/tcp comment 'loadweaver-traefik' || true + ufw allow proto 112 comment 'loadweaver-keepalived' || true +${listenerRules} + ufw --force enable || true +fi +`; +} + +export function buildHostBootstrapScript(os: HostOsInfo, options: HostProvisionOptions): string { + if (os.id !== 'debian' && os.id !== 'ubuntu') { + throw new Error(`Unsupported OS "${os.id}". Loadweaver host bootstrap supports Debian and Ubuntu only.`); + } + + const keepalivedBlock = options.installKeepalived + ? ` +if ! command -v keepalived >/dev/null 2>&1; then + apt-get install -y --no-install-recommends keepalived +fi +systemctl enable keepalived >/dev/null 2>&1 || true +` + : ''; + + const haproxyBlock = options.installHaproxy + ? ` +if ! command -v haproxy >/dev/null 2>&1; then + apt-get install -y --no-install-recommends haproxy +fi +systemctl enable haproxy >/dev/null 2>&1 || true +` + : ''; + + const birdBlock = options.installBird + ? ` +if ! command -v bird >/dev/null 2>&1; then + apt-get install -y --no-install-recommends bird2 +fi +systemctl enable bird >/dev/null 2>&1 || true +` + : ''; + + const cephadmBlock = options.installCephadm + ? ` +if ! command -v cephadm >/dev/null 2>&1; then + curl --fail --silent --show-error --remote-name --location \\ + "https://github.com/ceph/ceph/raw/${options.cephRelease}/src/cephadm/cephadm" + chmod +x cephadm + ./cephadm add-repo --release ${options.cephRelease} + ./cephadm install + rm -f cephadm +fi +` + : ''; + + return `#!/usr/bin/env bash +set -euo pipefail +export DEBIAN_FRONTEND=noninteractive +${aptProxyBlock(options.aptProxy)} +if [ "$(id -u)" -ne 0 ]; then + echo "Host bootstrap must run as root or via passwordless sudo." >&2 + exit 1 +fi + +apt-get update +apt-get install -y --no-install-recommends \\ + ca-certificates curl gnupg lsb-release \\ + apt-transport-https software-properties-common \\ + chrony lvm2 iproute2 iputils-ping ufw + +if ! command -v docker >/dev/null 2>&1; then + install -m 0755 -d /etc/apt/keyrings + curl -fsSL "https://download.docker.com/linux/${os.id}/gpg" -o /etc/apt/keyrings/docker.asc + chmod a+r /etc/apt/keyrings/docker.asc + echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/${os.id} ${os.versionCodename} stable" \\ + > /etc/apt/sources.list.d/docker.list + apt-get update + apt-get install -y --no-install-recommends \\ + docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin +fi + +systemctl enable --now docker + +if ! command -v wg >/dev/null 2>&1 || ! command -v wg-quick >/dev/null 2>&1; then + apt-get install -y --no-install-recommends wireguard wireguard-tools +fi +modprobe wireguard >/dev/null 2>&1 || true +${keepalivedBlock}${haproxyBlock}${birdBlock}${cephadmBlock}${firewallBlock( + options.configureFirewall, + options.wireguardPort, + options.listenerPorts ?? [], + )} +echo "loadweaver host bootstrap complete" +`; +} + +export interface HostSoftwareRequirements { + docker: boolean; + wireguard: boolean; + keepalived: boolean; + haproxy: boolean; + bird: boolean; + cephadm: boolean; +} + +export function buildHostVerificationScript(requirements: HostSoftwareRequirements): string { + const checks: string[] = ['command -v docker', 'docker info >/dev/null 2>&1', 'command -v wg', 'command -v wg-quick']; + + if (requirements.keepalived) { + checks.push('command -v keepalived'); + } + + if (requirements.haproxy) { + checks.push('command -v haproxy'); + } + + if (requirements.bird) { + checks.push('command -v bird'); + checks.push('command -v birdc'); + } + + if (requirements.cephadm) { + checks.push('command -v cephadm'); + } + + return checks.join(' && '); +} diff --git a/libs/domains/loadweaver/shared/feature-cli-host/src/lib/host.service.ts b/libs/domains/loadweaver/shared/feature-cli-host/src/lib/host.service.ts new file mode 100644 index 000000000..a7e338bfd --- /dev/null +++ b/libs/domains/loadweaver/shared/feature-cli-host/src/lib/host.service.ts @@ -0,0 +1,152 @@ +import type { LoadweaverContext } from '@forepath/loadweaver/shared/util-cli-core'; +import type { LoadweaverConfig } from '@forepath/loadweaver/shared/util-cli-core'; +import { deriveVipStateSnapshot, isRoutingHub, printStructuredOutput } from '@forepath/loadweaver/shared/util-cli-core'; + +import { + buildHostBootstrapScript, + buildHostVerificationScript, + parseOsRelease, + type HostSoftwareRequirements, +} from './host-provision-script'; + +function nodeHasCephRole(config: LoadweaverConfig, nodeId: string): boolean { + const roles = config.nodes[nodeId]?.roles ?? []; + return roles.some((role) => role.startsWith('ceph-')); +} + +function requirementsForNode(config: LoadweaverConfig, nodeId: string): HostSoftwareRequirements { + const vip = deriveVipStateSnapshot(config); + + return { + docker: true, + wireguard: true, + keepalived: vip.configured, + haproxy: vip.hasListeners, + bird: isRoutingHub(config, nodeId), + cephadm: nodeHasCephRole(config, nodeId), + }; +} + +export class HostService { + constructor(private readonly ctx: LoadweaverContext) {} + + async bootstrapAll(): Promise { + for (const nodeId of Object.keys(this.requireConfig().nodes)) { + await this.bootstrapNode(nodeId); + } + } + + async bootstrapNode(nodeId: string): Promise { + const config = this.requireConfig(); + + if (!config.nodes[nodeId]) { + throw new Error(`Unknown node: ${nodeId}`); + } + + const os = await this.detectOs(nodeId); + const vip = deriveVipStateSnapshot(config); + const script = buildHostBootstrapScript(os, { + installKeepalived: vip.configured, + installHaproxy: vip.hasListeners, + installBird: isRoutingHub(config, nodeId), + installCephadm: nodeHasCephRole(config, nodeId), + cephRelease: config.ceph.release, + wireguardPort: config.wireguard.port, + configureFirewall: config.host.configureFirewall, + aptProxy: config.host.aptProxy, + listenerPorts: vip.listenerPorts, + }); + + this.ctx.logger.info(`Bootstrapping host packages on ${nodeId} (${os.id} ${os.versionCodename})`); + await this.ctx.sshForNode(nodeId).execRemote(script, { dryRun: this.ctx.options.dryRun }); + } + + async verifyAll(): Promise { + if (this.ctx.options.dryRun) { + this.ctx.logger.warn('Skipping host software verification in dry-run mode'); + return; + } + + const failures: string[] = []; + + for (const nodeId of Object.keys(this.requireConfig().nodes)) { + const result = await this.verifyNode(nodeId); + + if (!result.passed) { + failures.push(`${nodeId}: ${result.message}`); + } + } + + if (failures.length > 0) { + throw new Error(`Host software verification failed:\n${failures.map((entry) => `- ${entry}`).join('\n')}`); + } + } + + async verifyNode(nodeId: string): Promise<{ passed: boolean; message: string }> { + const config = this.requireConfig(); + const requirements = requirementsForNode(config, nodeId); + const script = buildHostVerificationScript(requirements); + const result = await this.ctx + .sshForNode(nodeId) + .execRemote(`${script} && echo ok || echo missing`, { dryRun: this.ctx.options.dryRun }); + + if (result.stdout.trim() !== 'ok') { + return { + passed: false, + message: `Missing required host software (docker, wireguard${requirements.keepalived ? ', keepalived' : ''}${requirements.haproxy ? ', haproxy' : ''}${requirements.cephadm ? ', cephadm' : ''})`, + }; + } + + return { passed: true, message: 'Host software ready' }; + } + + async status(): Promise { + const nodes: Record = {}; + + for (const nodeId of Object.keys(this.requireConfig().nodes)) { + nodes[nodeId] = await this.verifyNode(nodeId); + } + + if (this.ctx.options.json) { + printStructuredOutput(this.ctx, { nodes }); + return; + } + + for (const [nodeId, result] of Object.entries(nodes)) { + this.ctx.logger.info(`${nodeId}: ${result.message}`); + } + } + + async bootstrapAndVerifyNode(nodeId: string): Promise { + await this.bootstrapNode(nodeId); + const readiness = await this.verifyNode(nodeId); + + if (!readiness.passed && !this.ctx.options.dryRun) { + throw new Error(`${nodeId}: ${readiness.message}`); + } + } + + private async detectOs(nodeId: string) { + if (this.ctx.options.dryRun) { + return { id: 'debian', versionCodename: 'bookworm' }; + } + + const result = await this.ctx + .sshForNode(nodeId) + .execRemote('cat /etc/os-release', { dryRun: this.ctx.options.dryRun }); + + if (!result.stdout.trim()) { + throw new Error(`Unable to read /etc/os-release from ${nodeId}`); + } + + return parseOsRelease(result.stdout); + } + + private requireConfig() { + if (!this.ctx.config) { + throw new Error('Configuration not loaded'); + } + + return this.ctx.config; + } +} diff --git a/libs/domains/loadweaver/shared/feature-cli-host/tsconfig.json b/libs/domains/loadweaver/shared/feature-cli-host/tsconfig.json new file mode 100644 index 000000000..9e5fde1c6 --- /dev/null +++ b/libs/domains/loadweaver/shared/feature-cli-host/tsconfig.json @@ -0,0 +1,16 @@ +{ + "extends": "../../../../../tsconfig.base.json", + "compilerOptions": { + "module": "commonjs" + }, + "files": [], + "include": [], + "references": [ + { + "path": "./tsconfig.lib.json" + }, + { + "path": "./tsconfig.spec.json" + } + ] +} diff --git a/libs/domains/loadweaver/shared/feature-cli-host/tsconfig.lib.json b/libs/domains/loadweaver/shared/feature-cli-host/tsconfig.lib.json new file mode 100644 index 000000000..841e950a0 --- /dev/null +++ b/libs/domains/loadweaver/shared/feature-cli-host/tsconfig.lib.json @@ -0,0 +1,15 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "../../../../../dist/out-tsc", + "declaration": true, + "types": ["node"] + }, + "include": ["src/**/*.ts"], + "exclude": [ + "jest.config.ts", + "jest.config.cts", + "src/**/*.spec.ts", + "src/**/*.test.ts" + ] +} diff --git a/libs/domains/loadweaver/shared/feature-cli-host/tsconfig.spec.json b/libs/domains/loadweaver/shared/feature-cli-host/tsconfig.spec.json new file mode 100644 index 000000000..67508cdda --- /dev/null +++ b/libs/domains/loadweaver/shared/feature-cli-host/tsconfig.spec.json @@ -0,0 +1,15 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "../../../../../dist/out-tsc", + "module": "commonjs", + "types": ["jest", "node"] + }, + "include": [ + "jest.config.ts", + "jest.config.cts", + "src/**/*.test.ts", + "src/**/*.spec.ts", + "src/**/*.d.ts" + ] +} diff --git a/libs/domains/loadweaver/shared/feature-cli-node/.eslintrc.json b/libs/domains/loadweaver/shared/feature-cli-node/.eslintrc.json new file mode 100644 index 000000000..b33995654 --- /dev/null +++ b/libs/domains/loadweaver/shared/feature-cli-node/.eslintrc.json @@ -0,0 +1,18 @@ +{ + "extends": ["../../../../../.eslintrc.json"], + "ignorePatterns": ["!**/*"], + "overrides": [ + { + "files": ["*.ts", "*.tsx", "*.js", "*.jsx"], + "rules": {} + }, + { + "files": ["*.ts", "*.tsx"], + "rules": {} + }, + { + "files": ["*.js", "*.jsx"], + "rules": {} + } + ] +} diff --git a/libs/domains/loadweaver/shared/feature-cli-node/README.md b/libs/domains/loadweaver/shared/feature-cli-node/README.md new file mode 100644 index 000000000..447939d5f --- /dev/null +++ b/libs/domains/loadweaver/shared/feature-cli-node/README.md @@ -0,0 +1,7 @@ +# loadweaver-shared-feature-cli-node + +This library was generated with [Nx](https://nx.dev). + +## Running unit tests + +Run `nx test loadweaver-shared-feature-cli-node` to execute the unit tests via [Jest](https://jestjs.io). diff --git a/libs/domains/loadweaver/shared/feature-cli-node/jest.config.cts b/libs/domains/loadweaver/shared/feature-cli-node/jest.config.cts new file mode 100644 index 000000000..56a3572a7 --- /dev/null +++ b/libs/domains/loadweaver/shared/feature-cli-node/jest.config.cts @@ -0,0 +1,11 @@ +module.exports = { + displayName: 'loadweaver-shared-feature-cli-node', + preset: '../../../../../jest.preset.cjs', + testEnvironment: 'node', + transform: { + '^.+\\.[tj]s$': ['ts-jest', { tsconfig: '/tsconfig.spec.json' }], + }, + moduleFileExtensions: ['ts', 'js', 'html'], + coverageDirectory: + '../../../../../coverage/libs/domains/loadweaver/shared/feature-cli-node', +}; diff --git a/libs/domains/loadweaver/shared/feature-cli-node/project.json b/libs/domains/loadweaver/shared/feature-cli-node/project.json new file mode 100644 index 000000000..8dea682e0 --- /dev/null +++ b/libs/domains/loadweaver/shared/feature-cli-node/project.json @@ -0,0 +1,16 @@ +{ + "name": "loadweaver-shared-feature-cli-node", + "$schema": "../../../../../node_modules/nx/schemas/project-schema.json", + "sourceRoot": "libs/domains/loadweaver/shared/feature-cli-node/src", + "projectType": "library", + "tags": ["domain:loadweaver", "scope:shared", "type:feature"], + "targets": { + "test": { + "executor": "@nx/jest:jest", + "outputs": ["{workspaceRoot}/coverage/{projectRoot}"], + "options": { + "jestConfig": "libs/domains/loadweaver/shared/feature-cli-node/jest.config.cts" + } + } + } +} diff --git a/libs/domains/loadweaver/shared/feature-cli-node/src/index.ts b/libs/domains/loadweaver/shared/feature-cli-node/src/index.ts new file mode 100644 index 000000000..2afb7ba9b --- /dev/null +++ b/libs/domains/loadweaver/shared/feature-cli-node/src/index.ts @@ -0,0 +1,2 @@ +export { registerNodeCommands } from './lib/command'; +export { NodeService } from './lib/node.service'; diff --git a/libs/domains/loadweaver/shared/feature-cli-node/src/lib/command.ts b/libs/domains/loadweaver/shared/feature-cli-node/src/lib/command.ts new file mode 100644 index 000000000..4861b391e --- /dev/null +++ b/libs/domains/loadweaver/shared/feature-cli-node/src/lib/command.ts @@ -0,0 +1,43 @@ +import type { Command } from 'commander'; + +import type { LoadweaverContext } from '@forepath/loadweaver/shared/util-cli-core'; +import { runGuardedMutation, withExamples } from '@forepath/loadweaver/shared/util-cli-core'; + +import { NodeService } from './node.service'; + +export function registerNodeCommands(program: Command, getCtx: (command: Command) => LoadweaverContext): void { + const node = program.command('node').description('Node lifecycle management'); + withExamples(node, ['loadweaver node join node-b1', 'loadweaver node label node-a1 site=a']); + + const join = node + .command('join') + .description('Join a node to the cluster') + .argument('', 'Node identifier') + .action(async function (this: Command, nodeId: string) { + const ctx = getCtx(this); + await runGuardedMutation(ctx, `node.join.${nodeId}`, () => new NodeService(ctx).join(nodeId)); + }); + withExamples(join, ['loadweaver node join node-b1']); + + const leave = node + .command('leave') + .description('Remove a node from the cluster') + .argument('', 'Node identifier') + .action(async function (this: Command, nodeId: string) { + const ctx = getCtx(this); + await runGuardedMutation(ctx, `node.leave.${nodeId}`, () => new NodeService(ctx).leave(nodeId)); + }); + withExamples(leave, ['loadweaver node leave node-b1']); + + const label = node + .command('label') + .description('Manage Swarm node labels') + .argument('', 'Node identifier') + .argument('