diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..db9c95f --- /dev/null +++ b/.env.example @@ -0,0 +1,39 @@ +# Copy this file to `.env` (same directory as docker-compose.yml) and adjust +# the values you want to change. Anything left out keeps the default baked +# into docker-compose.yml. +# +# Changing a value here updates it everywhere it's used — e.g. +# MONGO_ROOT_PASSWORD is shared by the api and mongodb services, so you only +# need to change it once, right here. + +# MongoDB root credentials, used by both the api and mongodb services. +# Changing MONGO_ROOT_PASSWORD after the database was initialized doesn't +# rotate it automatically — see docs.nodepit.com/runner/configuration. +MONGO_ROOT_USERNAME=root +MONGO_ROOT_PASSWORD=drnFEtX_PWKizUzKr3KT + +# Shared secret between the api and executor services. +API_KEY=nYCA3oaqhYJ__GEteWye + +# The URL Runner is reachable at. Change for anything beyond local use, e.g. +# https://npr.example.com — required when using the nginx extension. +WEB_BASE_URL=http://localhost:8080 + +# --- extensions/nginx --- +# Public domain name, must match WEB_BASE_URL above and have a DNS entry +# pointing at this server. +# DOMAIN=npr.example.com +# Email address used for Let's Encrypt certificate registration/renewal. +# LETSENCRYPT_EMAIL=admin@example.com + +# --- extensions/backup --- +# Local backups (./extensions/backup/archive) and the daily schedule/ +# retention work without any of these being set. Set the following to also +# ship backups to an S3-compatible bucket: +# BACKUP_S3_BUCKET=my-npr-backups +# BACKUP_S3_ACCESS_KEY_ID= +# BACKUP_S3_SECRET_ACCESS_KEY= +# BACKUP_S3_ENDPOINT=s3.amazonaws.com +# BACKUP_S3_PATH= +# BACKUP_CRON_EXPRESSION=0 3 * * * +# BACKUP_RETENTION_DAYS=7 diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml new file mode 100644 index 0000000..02a362e --- /dev/null +++ b/.github/workflows/validate.yml @@ -0,0 +1,40 @@ +name: Validate Compose + +on: + push: + pull_request: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + validate: + name: Validate (${{ matrix.name }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - name: base + files: -f docker-compose.yml + - name: nginx + files: -f docker-compose.yml -f extensions/nginx/docker-compose.yml + - name: watchtower + files: -f docker-compose.yml -f extensions/watchtower/docker-compose.yml + - name: backup + files: -f docker-compose.yml -f extensions/backup/docker-compose.yml + - name: portainer + files: -f docker-compose.yml -f extensions/portainer/docker-compose.yml + - name: all extensions combined + files: -f docker-compose.yml -f extensions/nginx/docker-compose.yml -f extensions/watchtower/docker-compose.yml -f extensions/backup/docker-compose.yml -f extensions/portainer/docker-compose.yml + env: + DOMAIN: example.com + LETSENCRYPT_EMAIL: admin@example.com + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - run: docker compose ${{ matrix.files }} config --quiet diff --git a/.gitignore b/.gitignore index 8000dd9..7fd771e 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,4 @@ .vagrant +.env +extensions/nginx/conf.d/default.conf +extensions/backup/archive/ diff --git a/README.md b/README.md index 9a27f8b..aff4da0 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,13 @@ + + + +
+ + + @@ -42,6 +49,25 @@ To upgrade an already running instance, run `docker compose pull` followed by `d The full manual — installation, configuration, guides and the API reference — lives at [**docs.nodepit.com/runner**](https://docs.nodepit.com/runner). +## 🧩 Extensions + +The [`extensions/`](extensions/) directory has optional add-ons, each a self-contained Compose file you layer on top of `docker-compose.yml` with `-f`. Some need their own settings — copy [`.env.example`](.env.example) to `.env` (next to `docker-compose.yml`) and fill in what’s needed: + +* [`extensions/nginx`](extensions/nginx) — HTTPS reverse proxy with automatic Let’s Encrypt certificates. +* [`extensions/watchtower`](extensions/watchtower) — automatically pulls and restarts containers when new images are released. +* [`extensions/backup`](extensions/backup) — daily database and workflow-data backups, stored locally and optionally shipped to an S3-compatible bucket. +* [`extensions/portainer`](extensions/portainer) — a web UI for managing the Docker stack without the CLI. + +Mix and match as needed, for example: + +```shell +docker compose \ + -f docker-compose.yml \ + -f extensions/nginx/docker-compose.yml \ + -f extensions/watchtower/docker-compose.yml \ + up -d +``` + ## 🤗 Get Involved Unsure if NodePit Runner is for you? Drop us a [mail](mailto:mail@nodepit.com) and we answer your questions and even better get you access to our cloud version of NodePit Runner for testing. @@ -58,7 +84,23 @@ NodePit Runner is licensed under the [NodePit Runner: Terms and Conditions](http
Show more - ## Vagrant + ### Changing secrets and settings + + `docker-compose.yml` works out of the box with built-in default secrets. To change one — e.g. the shared API key or a public `WEB_BASE_URL` — clone this repo, copy [`.env.example`](.env.example) to `.env`, and edit the values there. Because `.env` is read by every service that needs a given value, you only need to change it in one place. + + The MongoDB password is a special case: setting `MONGO_ROOT_PASSWORD` in `.env` only takes effect when the database is first initialized, not on a running installation. See [Configuration](https://docs.nodepit.com/runner/configuration) in the docs for how to rotate it safely. + + ### Combining extensions without repeating `-f` flags + + Instead of passing multiple `-f` flags every time, set `COMPOSE_FILE` in `.env`: + + ``` + COMPOSE_FILE=docker-compose.yml:extensions/nginx/docker-compose.yml:extensions/watchtower/docker-compose.yml + ``` + + and then just run `docker compose up -d`. + + ### Vagrant If you use [Vagrant](https://developer.hashicorp.com/vagrant), there’s a [Vagrantfile](Vagrantfile) to run a Debian box with Docker preinstalled. Start and connect to the box as follows: diff --git a/Vagrantfile b/Vagrantfile index 3c766e2..c730b2b 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -12,7 +12,7 @@ Vagrant.configure("2") do |config| # Every Vagrant development environment requires a box. You can search for # boxes at https://vagrantcloud.com/search. - config.vm.box = "debian/buster64" + config.vm.box = "debian/bookworm64" # Disable automatic box update checking. If you disable this, then # boxes will only be checked for updates when the user runs @@ -28,7 +28,16 @@ Vagrant.configure("2") do |config| # Create a forwarded port mapping which allows access to a specific port # within the machine from a port on the host machine and only allow access # via 127.0.0.1 to disable public access - config.vm.network "forwarded_port", guest: 8080, host: 8080, host_ip: "127.0.0.1" + # auto_correct falls back to a different free host port if the one + # requested is already taken, instead of failing `vagrant up` outright. + config.vm.network "forwarded_port", guest: 8080, host: 8080, host_ip: "127.0.0.1", auto_correct: true + + # Extension ports — only reachable if you're also running the + # corresponding extension inside the box: 80/443 for extensions/nginx, + # 9000 for extensions/portainer. + config.vm.network "forwarded_port", guest: 80, host: 80, host_ip: "127.0.0.1", auto_correct: true + config.vm.network "forwarded_port", guest: 443, host: 443, host_ip: "127.0.0.1", auto_correct: true + config.vm.network "forwarded_port", guest: 9000, host: 9000, host_ip: "127.0.0.1", auto_correct: true # Create a private network, which allows host-only access to the machine # using a specific IP. diff --git a/docker-compose.yml b/docker-compose.yml index 1a3bf0f..0a82435 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,14 @@ # NodePit Runner — Docker Compose file # -# Four services: api, web, executor and mongodb. Edit the values marked below, -# then run `docker compose up -d` to start the stack. +# Four services: api, web, executor and mongodb. Works out of the box with +# `docker compose up -d` — no editing required. +# +# All secrets and settings below have working defaults baked in via +# `${VAR:-default}`. To change one (e.g. the shared API key), place a `.env` +# file next to this one and set the variable there — see `.env.example`. +# +# Want HTTPS, automatic updates, backups, or a Docker management UI? See the +# `extensions/` directory and the README. # # Docs: # Installation https://docs.nodepit.com/runner/installation @@ -21,21 +28,24 @@ x-logging: services: api: image: ghcr.io/nodepit/runner-api:1 + restart: unless-stopped volumes: - api-data:/nodepit environment: - # Must match MONGO_INITDB_ROOT_USERNAME/PASSWORD on the mongodb service below. - DB_URL: mongodb://root:drnFEtX_PWKizUzKr3KT@mongodb/nodepit-runner?authSource=admin + # ⚠️ Public default — override in .env before production. + DB_URL: mongodb://${MONGO_ROOT_USERNAME:-root}:${MONGO_ROOT_PASSWORD:-drnFEtX_PWKizUzKr3KT}@mongodb/nodepit-runner?authSource=admin # Shared secret with API_KEY on the executor service below. - API_KEY: nYCA3oaqhYJ__GEteWye + # ⚠️ Public default — override in .env before production. + API_KEY: ${API_KEY:-nYCA3oaqhYJ__GEteWye} # The URL Runner is reachable at. Change for anything beyond local use. - WEB_BASE_URL: http://localhost:8080 + WEB_BASE_URL: ${WEB_BASE_URL:-http://localhost:8080} depends_on: - mongodb logging: *default-logging web: image: ghcr.io/nodepit/runner-web:1 + restart: unless-stopped ports: - 8080:80 environment: @@ -46,13 +56,14 @@ services: executor: image: ghcr.io/nodepit/runner-executor:1 + restart: unless-stopped volumes: - /var/run/docker.sock:/var/run/docker.sock - executor-data:/opt/data environment: API_BASE_URL: http://api:3000 # Shared secret with API_KEY on the api service above. - API_KEY: nYCA3oaqhYJ__GEteWye + API_KEY: ${API_KEY:-nYCA3oaqhYJ__GEteWye} depends_on: - api deploy: @@ -61,14 +72,15 @@ services: logging: *default-logging mongodb: - image: mongo:7.0.31 + image: mongo:7.0.37 + restart: unless-stopped volumes: - mongodb-data:/data/db environment: - # Must match the username/password in DB_URL on the api service above. - # Only used when initializing a new database — see the docs to change it later. - MONGO_INITDB_ROOT_USERNAME: root - MONGO_INITDB_ROOT_PASSWORD: drnFEtX_PWKizUzKr3KT + # Only used when initializing a new database — see docs to change it later. + # ⚠️ Public default — override in .env before production. + MONGO_INITDB_ROOT_USERNAME: ${MONGO_ROOT_USERNAME:-root} + MONGO_INITDB_ROOT_PASSWORD: ${MONGO_ROOT_PASSWORD:-drnFEtX_PWKizUzKr3KT} logging: *default-logging volumes: diff --git a/extensions/backup/docker-compose.yml b/extensions/backup/docker-compose.yml new file mode 100644 index 0000000..e7cca8d --- /dev/null +++ b/extensions/backup/docker-compose.yml @@ -0,0 +1,48 @@ +# NodePit Runner — backup extension +# +# Backs up the mongodb database (via mongodump, so it's consistent even +# while Runner is running) and the api-data volume (uploaded workflows and +# environments) on a daily schedule. The executor-data volume is excluded — +# it's the executor's working/cache directory, not source-of-truth data. +# +# By default, backups are written locally to ./extensions/backup/archive on +# the host. To also (or instead) ship them off-site, set the AWS_S3_* / S3_* +# variables in .env — see .env.example. Uses +# https://offen.github.io/docker-volume-backup/ under the hood. +# +# Usage: +# docker compose -f docker-compose.yml -f extensions/backup/docker-compose.yml up -d + +services: + mongodb: + volumes: + - mongodb-dump:/dump + labels: + docker-volume-backup.archive-pre: >- + /bin/sh -c 'mongodump --archive --gzip + --uri="mongodb://${MONGO_ROOT_USERNAME:-root}:${MONGO_ROOT_PASSWORD:-drnFEtX_PWKizUzKr3KT}@localhost/nodepit-runner?authSource=admin" + > /dump/mongodb.archive.gz' + docker-volume-backup.archive-post: "rm -f /dump/mongodb.archive.gz" + + backup: + image: offen/docker-volume-backup:v2.48.2 + restart: unless-stopped + environment: + BACKUP_CRON_EXPRESSION: ${BACKUP_CRON_EXPRESSION:-0 3 * * *} + BACKUP_FILENAME: npr-backup-%Y-%m-%dT%H-%M-%S.tar.gz + BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7} + # Leave unset to skip shipping backups off-site — local backups under + # ./extensions/backup/archive still happen either way. + AWS_S3_BUCKET_NAME: ${BACKUP_S3_BUCKET:-} + AWS_S3_PATH: ${BACKUP_S3_PATH:-} + AWS_ACCESS_KEY_ID: ${BACKUP_S3_ACCESS_KEY_ID:-} + AWS_SECRET_ACCESS_KEY: ${BACKUP_S3_SECRET_ACCESS_KEY:-} + AWS_ENDPOINT: ${BACKUP_S3_ENDPOINT:-s3.amazonaws.com} + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + - mongodb-dump:/backup/mongodb-dump:ro + - api-data:/backup/api-data:ro + - ./extensions/backup/archive:/archive + +volumes: + mongodb-dump: diff --git a/extensions/nginx/conf.d/client_max_body_size.conf b/extensions/nginx/conf.d/client_max_body_size.conf new file mode 100644 index 0000000..9950721 --- /dev/null +++ b/extensions/nginx/conf.d/client_max_body_size.conf @@ -0,0 +1,3 @@ +# disable nginx file upload limit which is per default `1m` +# https://nginx.org/en/docs/http/ngx_http_core_module.html +client_max_body_size 0; diff --git a/extensions/nginx/docker-compose.yml b/extensions/nginx/docker-compose.yml new file mode 100644 index 0000000..36c8f3f --- /dev/null +++ b/extensions/nginx/docker-compose.yml @@ -0,0 +1,50 @@ +# NodePit Runner — nginx extension +# +# Adds an HTTPS reverse proxy (nginx-proxy) in front of the web/api services, +# with automatic Let's Encrypt certificates via acme-companion. +# +# Requires DOMAIN and LETSENCRYPT_EMAIL to be set in .env (see +# .env.example), and a DNS entry pointing DOMAIN at this server. Also set +# WEB_BASE_URL=https:// in .env so links generated by Runner use the +# public HTTPS URL. +# +# Usage: +# docker compose -f docker-compose.yml -f extensions/nginx/docker-compose.yml up -d + +services: + web: + environment: + VIRTUAL_HOST: ${DOMAIN:?Set DOMAIN in .env before using the nginx extension — see .env.example} + LETSENCRYPT_HOST: ${DOMAIN:?Set DOMAIN in .env before using the nginx extension — see .env.example} + LETSENCRYPT_EMAIL: ${LETSENCRYPT_EMAIL:?Set LETSENCRYPT_EMAIL in .env before using the nginx extension — see .env.example} + + nginx-proxy: + image: nginxproxy/nginx-proxy:1.11.5 + restart: unless-stopped + ports: + - 80:80 + - 443:443 + volumes: + - certs:/etc/nginx/certs + - vhost:/etc/nginx/vhost.d + - html:/usr/share/nginx/html + - /var/run/docker.sock:/tmp/docker.sock:ro + - ./extensions/nginx/conf.d:/etc/nginx/conf.d + labels: + - com.github.nginx-proxy.nginx=true + + acme-companion: + image: nginxproxy/acme-companion:2.8.1 + restart: unless-stopped + volumes: + - certs:/etc/nginx/certs + - vhost:/etc/nginx/vhost.d + - html:/usr/share/nginx/html + - acme:/etc/acme.sh + - /var/run/docker.sock:/var/run/docker.sock:ro + +volumes: + certs: + vhost: + html: + acme: diff --git a/extensions/portainer/docker-compose.yml b/extensions/portainer/docker-compose.yml new file mode 100644 index 0000000..240f77e --- /dev/null +++ b/extensions/portainer/docker-compose.yml @@ -0,0 +1,29 @@ +# NodePit Runner — portainer extension +# +# Adds Portainer CE, a web UI for managing the Docker stack (start/stop +# containers, view logs, inspect resource usage) without needing the CLI. +# On first visit, it asks you to create an admin account. +# +# Bound to 127.0.0.1 only — with the Docker socket mounted, anyone who can +# log in has root-equivalent access to the host, so it's not exposed +# publicly by default. On a remote host, reach it via an SSH tunnel: +# ssh -L 9000:localhost:9000 user@host +# The nginx extension doesn't proxy Portainer, only web/api — combining +# the two requires extending extensions/nginx yourself. +# +# Usage: +# docker compose -f docker-compose.yml -f extensions/portainer/docker-compose.yml up -d +# Open http://localhost:9000 + +services: + portainer: + image: portainer/portainer-ce:2.39.5 + restart: unless-stopped + ports: + - 127.0.0.1:9000:9000 + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - portainer-data:/data + +volumes: + portainer-data: diff --git a/extensions/watchtower/docker-compose.yml b/extensions/watchtower/docker-compose.yml new file mode 100644 index 0000000..afb5820 --- /dev/null +++ b/extensions/watchtower/docker-compose.yml @@ -0,0 +1,27 @@ +# NodePit Runner — watchtower extension +# +# Automatically keeps the NodePit Runner containers up to date by pulling +# and restarting them on a daily interval. +# +# Usage: +# docker compose -f docker-compose.yml -f extensions/watchtower/docker-compose.yml up -d + +services: + api: + labels: + - com.centurylinklabs.watchtower.enable=true + + web: + labels: + - com.centurylinklabs.watchtower.enable=true + + executor: + labels: + - com.centurylinklabs.watchtower.enable=true + + watchtower: + image: nickfedor/watchtower:1.19.0 + restart: unless-stopped + volumes: + - /var/run/docker.sock:/var/run/docker.sock + command: --interval 86400 --label-enable --cleanup