diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f2649c9..cc5c653 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -91,35 +91,36 @@ jobs: targets: wasm32-unknown-unknown - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 - - name: Start MinIO + - name: Start RustFS run: | - docker run -d --name minio \ + docker run -d --name rustfs \ -p 9000:9000 \ - -e MINIO_ROOT_USER=minioadmin \ - -e MINIO_ROOT_PASSWORD=minioadmin \ - minio/minio:latest server /data + -e RUSTFS_ACCESS_KEY=minioadmin \ + -e RUSTFS_SECRET_KEY=minioadmin \ + rustfs/rustfs:1.0.0 - # Wait for MinIO to be ready + # Wait for RustFS to be ready for i in $(seq 1 30); do - if curl -sf http://localhost:9000/minio/health/live > /dev/null 2>&1; then - echo "MinIO is ready" + if curl -sf http://localhost:9000/health > /dev/null 2>&1; then + echo "RustFS is ready" break fi sleep 1 done - - name: Seed MinIO buckets + - name: Seed buckets + env: + AWS_ACCESS_KEY_ID: minioadmin + AWS_SECRET_ACCESS_KEY: minioadmin + AWS_DEFAULT_REGION: us-east-1 run: | - curl -sSL https://dl.min.io/client/mc/release/linux-amd64/mc -o /usr/local/bin/mc - chmod +x /usr/local/bin/mc - - mc alias set local http://localhost:9000 minioadmin minioadmin - mc mb --ignore-existing local/public-data - mc mb --ignore-existing local/private-uploads - mc anonymous set download local/public-data - echo "Hello from s3-proxy!" | mc pipe local/public-data/hello.txt - echo '{"status":"ok"}' | mc pipe local/public-data/health.json - echo "Secret payload" | mc pipe local/private-uploads/docs/secret.txt + s3() { aws --endpoint-url http://localhost:9000 s3 "$@"; } + + s3 mb s3://public-data + s3 mb s3://private-uploads + echo "Hello from s3-proxy!" | s3 cp - s3://public-data/hello.txt + echo '{"status":"ok"}' | s3 cp - s3://public-data/health.json + echo "Secret payload" | s3 cp - s3://private-uploads/docs/secret.txt - name: Build worker working-directory: examples/cf-workers diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fe971b9..8ab17e3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -20,7 +20,7 @@ cargo check -p multistore-cf-workers --target wasm32-unknown-unknown # Unit + doc tests cargo test -# Integration tests (MinIO via docker compose behind wrangler dev; mirrors CI) +# Integration tests (RustFS via docker compose behind wrangler dev; mirrors CI) make test-integration ``` diff --git a/Makefile b/Makefile index 9c39ed4..3e110f0 100644 --- a/Makefile +++ b/Makefile @@ -24,7 +24,7 @@ clippy-fix: test: cargo test -# Run the integration suite locally: MinIO (docker compose) + the Workers +# Run the integration suite locally: RustFS (docker compose) + the Workers # runtime (wrangler dev), mirroring CI. Pass extra pytest args via ARGS. test-integration: ./scripts/integration-test.sh $(ARGS) diff --git a/docker-compose.yaml b/docker-compose.yaml index ab70ce8..9800898 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,39 +1,42 @@ services: - # ── MinIO (backing object store) ────────────────────────────────────── - minio: - image: minio/minio:latest - command: server /data --console-address ":9001" + # ── RustFS (backing object store) ───────────────────────────────────── + rustfs: + image: rustfs/rustfs:1.0.0 ports: - "9000:9000" # S3 API - "9001:9001" # Web console environment: - MINIO_ROOT_USER: minioadmin - MINIO_ROOT_PASSWORD: minioadmin + RUSTFS_ACCESS_KEY: minioadmin + RUSTFS_SECRET_KEY: minioadmin volumes: - - minio-data:/data + - rustfs-data:/data healthcheck: - test: ["CMD", "mc", "ready", "local"] + test: ["CMD", "curl", "-sf", "http://localhost:9000/health"] interval: 5s timeout: 5s retries: 5 - # ── Seed MinIO with example buckets and data ────────────────────────── - minio-init: - image: minio/mc:latest + # ── Seed RustFS with example buckets and data ───────────────────────── + rustfs-init: + image: amazon/aws-cli:2.37.2 depends_on: - minio: + rustfs: condition: service_healthy + environment: + AWS_ACCESS_KEY_ID: minioadmin + AWS_SECRET_ACCESS_KEY: minioadmin + AWS_DEFAULT_REGION: us-east-1 entrypoint: /bin/sh command: - -c - | - mc alias set local http://minio:9000 minioadmin minioadmin - mc mb --ignore-existing local/public-data - mc mb --ignore-existing local/private-uploads - mc anonymous set download local/public-data - echo "Hello from s3-proxy!" | mc pipe local/public-data/hello.txt - echo '{"status":"ok"}' | mc pipe local/public-data/health.json - echo "Secret payload" | mc pipe local/private-uploads/docs/secret.txt + s3() { aws --endpoint-url http://rustfs:9000 s3 "$$@"; } + for b in public-data private-uploads; do + s3 ls "s3://$$b" >/dev/null 2>&1 || s3 mb "s3://$$b" + done + echo "Hello from s3-proxy!" | s3 cp - s3://public-data/hello.txt + echo '{"status":"ok"}' | s3 cp - s3://public-data/health.json + echo "Secret payload" | s3 cp - s3://private-uploads/docs/secret.txt volumes: - minio-data: \ No newline at end of file + rustfs-data: diff --git a/docs/getting-started/index.md b/docs/getting-started/index.md index 2c5a27f..0055fc6 100644 --- a/docs/getting-started/index.md +++ b/docs/getting-started/index.md @@ -5,24 +5,24 @@ The Multistore proxy is a multi-runtime S3 gateway that proxies requests to back ## Prerequisites - [Rust](https://www.rust-lang.org/tools/install) (latest stable) -- [Docker](https://docs.docker.com/get-docker/) (for local development with MinIO) +- [Docker](https://docs.docker.com/get-docker/) (for local development with RustFS) ## Start the Backend -Use Docker Compose to start MinIO as a local object store: +Use Docker Compose to start RustFS as a local object store: ```bash docker compose up ``` This starts: -- MinIO API on port `9000` -- MinIO Console on port `9001` (user: `minioadmin`, password: `minioadmin`) +- RustFS API on port `9000` +- RustFS Console on port `9001` (user: `minioadmin`, password: `minioadmin`) - A seed job that creates example buckets with test data ## Run the Proxy -The shipped Cloudflare Workers config (`examples/cf-workers/wrangler.toml`) wires the `public-data` and `private-uploads` buckets to the local MinIO instance started above, so it works out of the box: +The shipped Cloudflare Workers config (`examples/cf-workers/wrangler.toml`) wires the `public-data` and `private-uploads` buckets to the local RustFS instance started above, so it works out of the box: ```bash cd examples/cf-workers && npx wrangler dev @@ -30,7 +30,7 @@ cd examples/cf-workers && npx wrangler dev The Workers dev server listens on port `8787`. -> The native server example (`examples/server/config.toml`) ships a different set of buckets pointed at real AWS S3, not the local MinIO buckets. See [Local Development](./local-development) for details on each runtime's config. +> The native server example (`examples/server/config.toml`) ships a different set of buckets pointed at real AWS S3, not the local RustFS buckets. See [Local Development](./local-development) for details on each runtime's config. ## Make Your First Request diff --git a/docs/getting-started/local-development.md b/docs/getting-started/local-development.md index 5ac8308..a7644d7 100644 --- a/docs/getting-started/local-development.md +++ b/docs/getting-started/local-development.md @@ -1,18 +1,18 @@ # Local Development -This guide walks through setting up a full local development environment with MinIO as the backing object store. +This guide walks through setting up a full local development environment with [RustFS](https://github.com/rustfs/rustfs), an S3-compatible server, as the backing object store. ## Docker Compose -The project includes a `docker-compose.yaml` that starts MinIO and seeds it with example data: +The project includes a `docker-compose.yaml` that starts RustFS and seeds it with example data: ```bash docker compose up ``` This starts: -- **MinIO API** at `http://localhost:9000` -- **MinIO Console** at `http://localhost:9001` (credentials: `minioadmin` / `minioadmin`) +- **RustFS S3 API** at `http://localhost:9000` +- **RustFS Console** at `http://localhost:9001` (credentials: `minioadmin` / `minioadmin`) - A seed job that creates `public-data` and `private-uploads` buckets with sample files ## Configuration Files @@ -21,17 +21,17 @@ The two runtimes use different config formats: ### Workers Runtime — `examples/cf-workers/wrangler.toml` -The CF Workers runtime reads `PROXY_CONFIG` from the Wrangler configuration. The shipped `wrangler.toml` points the `public-data` and `private-uploads` buckets at `http://localhost:9000` (the MinIO instance seeded by Docker Compose), so it works against the local backend out of the box: +The CF Workers runtime reads `PROXY_CONFIG` from the Wrangler configuration. The shipped `wrangler.toml` points the `public-data` and `private-uploads` buckets at `http://localhost:9000` (the RustFS instance seeded by Docker Compose), so it works against the local backend out of the box: ```bash cd examples/cf-workers && npx wrangler dev ``` -The Workers dev server runs on port `8787` by default. This is the recommended runtime for the local MinIO quickstart. +The Workers dev server runs on port `8787` by default. This is the recommended runtime for the local RustFS quickstart. ### Server Runtime — `examples/server/config.toml` -The server runtime reads a TOML config file, defaulting to `config.toml` in the working directory. The shipped `examples/server/config.toml` serves the `harvard-lil` and `cholmes` buckets from real AWS S3 (not the local MinIO buckets), so point `--config` at your own config to use a local backend: +The server runtime reads a TOML config file, defaulting to `config.toml` in the working directory. The shipped `examples/server/config.toml` serves the `harvard-lil` and `cholmes` buckets from real AWS S3 (not the local RustFS buckets), so point `--config` at your own config to use a local backend: ```bash cargo run -p multistore-server -- \ @@ -66,7 +66,7 @@ For local development, these are optional but useful: ## Verifying the Setup -Once the Workers dev server is running, test both anonymous and authenticated access against the local MinIO buckets: +Once the Workers dev server is running, test both anonymous and authenticated access against the local RustFS buckets: ```bash # Anonymous read (should return file contents) @@ -84,6 +84,6 @@ AWS_SECRET_ACCESS_KEY="localdev/secret/key/00000000000000000000" \ aws s3 ls s3://private-uploads/ \ --endpoint-url http://localhost:8787 -# Browse MinIO directly +# Browse RustFS directly open http://localhost:9001 ``` diff --git a/scripts/integration-test.sh b/scripts/integration-test.sh index f68edf1..6c0a180 100755 --- a/scripts/integration-test.sh +++ b/scripts/integration-test.sh @@ -1,13 +1,13 @@ #!/usr/bin/env bash # # Run the integration test suite locally — the same setup CI uses: -# MinIO (via docker compose) behind the Cloudflare Workers runtime (wrangler dev). +# RustFS (via docker compose) behind the Cloudflare Workers runtime (wrangler dev). # # Usage: # ./scripts/integration-test.sh [extra pytest args] # make test-integration # -# MinIO lifecycle: if MinIO isn't running, this starts it and stops it again on +# RustFS lifecycle: if RustFS isn't running, this starts it and stops it again on # exit (clean one-off run). If it's already up — e.g. you ran `docker compose # up -d` to iterate — it's left running so repeated runs stay fast. wrangler dev # is always stopped on exit. @@ -23,38 +23,38 @@ PORT="${PROXY_PORT:-8787}" PROXY_URL="http://localhost:${PORT}" WORKER_DIR="examples/cf-workers" -# Tear down MinIO on exit only if we started it. If it was already running +# Tear down RustFS on exit only if we started it. If it was already running # (e.g. a dev who ran `docker compose up -d` to iterate), leave it warm so # repeated runs stay fast and we don't stop a stack we didn't start. -STARTED_MINIO=false +STARTED_BACKEND=false cleanup() { # Kill wrangler (and the workerd children it spawns). pkill -f "wrangler dev --config wrangler.integration.toml" 2>/dev/null || true - if [ "$STARTED_MINIO" = true ]; then - echo "==> Stopping MinIO (started by this run)" + if [ "$STARTED_BACKEND" = true ]; then + echo "==> Stopping RustFS (started by this run)" docker compose down >/dev/null 2>&1 || true fi } trap cleanup EXIT -if curl -sf -o /dev/null http://localhost:9000/minio/health/live 2>/dev/null; then - echo "==> MinIO already running — leaving it up after the run" +if curl -sf -o /dev/null http://localhost:9000/health 2>/dev/null; then + echo "==> RustFS already running — leaving it up after the run" else - STARTED_MINIO=true + STARTED_BACKEND=true fi -echo "==> Starting MinIO + seeding buckets (docker compose)" -# Not `--wait`: the one-shot seeder (minio-init) exits, which `--wait` treats as -# a failure. Instead poll for a seeded public object — that confirms MinIO is up +echo "==> Starting RustFS + seeding buckets (docker compose)" +# Not `--wait`: the one-shot seeder (rustfs-init) exits, which `--wait` treats as +# a failure. Instead poll for a seeded public object — that confirms RustFS is up # *and* the buckets are seeded. docker compose up -d for i in $(seq 1 30); do if curl -so /dev/null "http://localhost:9000/public-data/hello.txt"; then - echo " MinIO ready and seeded" + echo " RustFS ready and seeded" break fi if [ "$i" -eq 30 ]; then - echo "ERROR: MinIO did not become ready within 30s" >&2 + echo "ERROR: RustFS did not become ready within 30s" >&2 exit 1 fi sleep 1