From b168bb4b707dac55ac859829ca8e924c74a913a8 Mon Sep 17 00:00:00 2001 From: Noah Treuhaft Date: Mon, 14 Sep 2026 13:07:47 -0400 Subject: [PATCH 1/5] replace MinIO with RustFS in system tests MinIO is no longer available for download. --- Makefile | 21 +++++++++++++-------- cmd/super/ztests/s3-multiple.yaml | 10 ++++++---- cmd/super/ztests/s3-notexist.yaml | 7 ++++--- cmd/super/ztests/s3-parquet.yaml | 9 +++++---- cmd/super/ztests/s3-simple.yaml | 9 +++++---- db/ztests/s3/rm.yaml | 7 ++++--- db/ztests/s3/stat.yaml | 7 ++++--- db/ztests/s3/zq.yaml | 7 ++++--- testdata/{minio.sh => s3.sh} | 17 ++++++++++------- 9 files changed, 55 insertions(+), 39 deletions(-) rename testdata/{minio.sh => s3.sh} (52%) diff --git a/Makefile b/Makefile index 95142f7981..39ce5c2127 100644 --- a/Makefile +++ b/Makefile @@ -34,11 +34,6 @@ tidy: go mod tidy git diff --exit-code -- go.mod go.sum -bin/minio: Makefile - @curl -fLo $@ --compressed --create-dirs \ - https://dl.min.io/server/minio/release/$$(go env GOOS)-$$(go env GOARCH)/archive/minio.RELEASE.2022-05-04T07-45-27Z - @chmod +x $@ - generate: go generate ./... @@ -48,18 +43,28 @@ test-generate: generate test-unit: @go test -short ./... -test-system: build bin/minio +test-system: build bin/rustfs @ZTEST_PATH="$(CURDIR)/dist:$(CURDIR)/bin" go test . -test-boomerangs: build bin/minio +test-boomerangs: build bin/rustfs @ZTEST_PATH="$(CURDIR)/dist:$(CURDIR)/bin" go test -parallel 1 . -run TestSPQ/boomerang -v -test-run: build bin/minio +test-run: build bin/rustfs @ZTEST_PATH="$(CURDIR)/dist:$(CURDIR)/bin" go test . -v -run $(TEST) test-heavy: build @PATH="$(CURDIR)/dist:$(PATH)" go test -tags=heavy ./mdtest +bin/rustfs: Makefile + arch=$$(uname -m | sed -e s/^arm/aarch/) \ + os=$$(uname -s | sed -e s/^Darwin/macos/ | tr A-Z a-z) && \ + [ $$os = linux ] && libc=-gnu && \ + url=https://github.com/rustfs/rustfs/releases/download/1.0.0-rc.5/rustfs-$${os}-$${arch}$${libc}-latest.zip && \ + echo $$url && \ + curl -Lf --create-dirs -o $@.zip $$url + unzip -oq $@.zip -d $(@D) + touch -c $@ + build: $(PEG_DEP) @mkdir -p dist @go build -ldflags='$(LDFLAGS)' -o dist ./cmd/... diff --git a/cmd/super/ztests/s3-multiple.yaml b/cmd/super/ztests/s3-multiple.yaml index 0caab26b80..e876c7984b 100644 --- a/cmd/super/ztests/s3-multiple.yaml +++ b/cmd/super/ztests/s3-multiple.yaml @@ -1,11 +1,13 @@ script: | - source minio.sh - mv *.sup data/bucket + source s3.sh + aws s3api create-bucket --bucket bucket + aws s3 cp --quiet log1.sup s3://bucket + aws s3 cp --quiet log2.sup s3://bucket super -s -c "sort ts" s3://bucket/log1.sup s3://bucket/log2.sup inputs: - - name: minio.sh - source: ../../../testdata/minio.sh + - name: s3.sh + source: ../../../testdata/s3.sh - name: log1.sup data: | {ts:2018-03-24T17:15:21.255387Z,uid:"C8Tful1TvM3Zf5x8fl"} diff --git a/cmd/super/ztests/s3-notexist.yaml b/cmd/super/ztests/s3-notexist.yaml index 6c3a2653d0..348bd2649b 100644 --- a/cmd/super/ztests/s3-notexist.yaml +++ b/cmd/super/ztests/s3-notexist.yaml @@ -1,10 +1,11 @@ script: | - source minio.sh + source s3.sh + aws s3api create-bucket --bucket bucket ! super -s s3://bucket/does/not/exist inputs: - - name: minio.sh - source: ../../../testdata/minio.sh + - name: s3.sh + source: ../../../testdata/s3.sh outputs: - name: stderr diff --git a/cmd/super/ztests/s3-parquet.yaml b/cmd/super/ztests/s3-parquet.yaml index 321f0265a4..77890d8b2c 100644 --- a/cmd/super/ztests/s3-parquet.yaml +++ b/cmd/super/ztests/s3-parquet.yaml @@ -1,11 +1,12 @@ script: | - source minio.sh - mv dns.parquet data/bucket + source s3.sh + aws s3api create-bucket --bucket bucket + aws s3 cp --quiet dns.parquet s3://bucket super -s -c "count()" s3://bucket/dns.parquet inputs: - - name: minio.sh - source: ../../../testdata/minio.sh + - name: s3.sh + source: ../../../testdata/s3.sh - name: dns.parquet source: ../../../sio/parquetio/ztests/dns.parquet diff --git a/cmd/super/ztests/s3-simple.yaml b/cmd/super/ztests/s3-simple.yaml index 1a161be574..3571303cb0 100644 --- a/cmd/super/ztests/s3-simple.yaml +++ b/cmd/super/ztests/s3-simple.yaml @@ -1,13 +1,14 @@ script: | - source minio.sh - mv babble.sup ./data/bucket + source s3.sh + aws s3api create-bucket --bucket bucket + aws s3 cp --quiet babble.sup s3://bucket super -s -c "count()" s3://bucket/babble.sup inputs: - name: babble.sup source: ../../../testdata/babble.sup - - name: minio.sh - source: ../../../testdata/minio.sh + - name: s3.sh + source: ../../../testdata/s3.sh outputs: - name: stdout diff --git a/db/ztests/s3/rm.yaml b/db/ztests/s3/rm.yaml index 8a7fcdba1e..92141afd79 100644 --- a/db/ztests/s3/rm.yaml +++ b/db/ztests/s3/rm.yaml @@ -2,7 +2,8 @@ skip: issue 2540... need to delete by key range script: | mkdir logs - source minio.sh + source s3.sh + aws s3api create-bucket --bucket bucket super db import -R logs -data s3://bucket/db babble.sup super db map -R logs -o count.bsup "count()" echo === @@ -15,8 +16,8 @@ script: | inputs: - name: babble.sup source: ../../../testdata/babble.sup - - name: minio.sh - source: ../../../testdata/minio.sh + - name: s3.sh + source: ../../../testdata/s3.sh outputs: - name: stdout diff --git a/db/ztests/s3/stat.yaml b/db/ztests/s3/stat.yaml index ca5d12669f..d5f7bcb41e 100644 --- a/db/ztests/s3/stat.yaml +++ b/db/ztests/s3/stat.yaml @@ -1,5 +1,6 @@ script: | - source minio.sh + source s3.sh + aws s3api create-bucket --bucket bucket export SUPER_DB=s3://bucket/db super db init -q super db create -q logs @@ -9,8 +10,8 @@ script: | inputs: - name: babble.sup source: ../../../testdata/babble.sup - - name: minio.sh - source: ../../../testdata/minio.sh + - name: s3.sh + source: ../../../testdata/s3.sh outputs: - name: stdout diff --git a/db/ztests/s3/zq.yaml b/db/ztests/s3/zq.yaml index fbc57e4407..2314fa23d6 100644 --- a/db/ztests/s3/zq.yaml +++ b/db/ztests/s3/zq.yaml @@ -1,5 +1,6 @@ script: | - source minio.sh + source s3.sh + aws s3api create-bucket --bucket bucket export SUPER_DB=s3://bucket/db super db init -q super db create -q logs @@ -9,8 +10,8 @@ script: | inputs: - name: babble.sup source: ../../../testdata/babble.sup - - name: minio.sh - source: ../../../testdata/minio.sh + - name: s3.sh + source: ../../../testdata/s3.sh outputs: - name: stdout diff --git a/testdata/minio.sh b/testdata/s3.sh similarity index 52% rename from testdata/minio.sh rename to testdata/s3.sh index 153dfa50ce..df4937db20 100644 --- a/testdata/minio.sh +++ b/testdata/s3.sh @@ -1,14 +1,18 @@ #!/bin/bash -mkdir -p data/bucket +export AWS_ACCESS_KEY_ID=admin +export AWS_SECRET_ACCESS_KEY=secret -# Allocate a port. Another process could bind to it before MinIO does, +# Allocate a port. Another process could bind to it before ours does, # but that's very unlikely. port=$(python3 -c "import socket; print(socket.create_server(('localhost', 0)).getsockname()[1])") -minio server --address localhost:$port --console-address localhost:0 --quiet data & + +dir=$PWD/s3 +mkdir $dir +rustfs server $dir --address localhost:$port --access-key $AWS_ACCESS_KEY_ID --secret-key $AWS_SECRET_ACCESS_KEY & trap "kill -9 $!" EXIT -# Wait for MinIO to accept a connection. +# Wait for server to accept a connection. python3 < Date: Mon, 14 Sep 2026 13:16:19 -0400 Subject: [PATCH 2/5] try musl --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 39ce5c2127..a293c5d261 100644 --- a/Makefile +++ b/Makefile @@ -58,7 +58,7 @@ test-heavy: build bin/rustfs: Makefile arch=$$(uname -m | sed -e s/^arm/aarch/) \ os=$$(uname -s | sed -e s/^Darwin/macos/ | tr A-Z a-z) && \ - [ $$os = linux ] && libc=-gnu && \ + [ $$os = linux ] && libc=-musl && \ url=https://github.com/rustfs/rustfs/releases/download/1.0.0-rc.5/rustfs-$${os}-$${arch}$${libc}-latest.zip && \ echo $$url && \ curl -Lf --create-dirs -o $@.zip $$url From b6b4f317eed6ae3dcdf1c3db6944f92a2e253ffa Mon Sep 17 00:00:00 2001 From: Noah Treuhaft Date: Mon, 14 Sep 2026 13:35:36 -0400 Subject: [PATCH 3/5] fix some problems --- Makefile | 2 +- testdata/s3.sh | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index a293c5d261..174184a94c 100644 --- a/Makefile +++ b/Makefile @@ -58,7 +58,7 @@ test-heavy: build bin/rustfs: Makefile arch=$$(uname -m | sed -e s/^arm/aarch/) \ os=$$(uname -s | sed -e s/^Darwin/macos/ | tr A-Z a-z) && \ - [ $$os = linux ] && libc=-musl && \ + [ $$os != linux ] || libc=-musl && \ url=https://github.com/rustfs/rustfs/releases/download/1.0.0-rc.5/rustfs-$${os}-$${arch}$${libc}-latest.zip && \ echo $$url && \ curl -Lf --create-dirs -o $@.zip $$url diff --git a/testdata/s3.sh b/testdata/s3.sh index df4937db20..965833cc8f 100644 --- a/testdata/s3.sh +++ b/testdata/s3.sh @@ -9,7 +9,8 @@ port=$(python3 -c "import socket; print(socket.create_server(('localhost', 0)).g dir=$PWD/s3 mkdir $dir -rustfs server $dir --address localhost:$port --access-key $AWS_ACCESS_KEY_ID --secret-key $AWS_SECRET_ACCESS_KEY & +RUSTFS_OBS_LOG_DIRECTORY=rustfs rustfs server $dir \ + --address localhost:$port --access-key $AWS_ACCESS_KEY_ID --secret-key $AWS_SECRET_ACCESS_KEY & trap "kill -9 $!" EXIT # Wait for server to accept a connection. From 4a418f94a19d3d107276196ff599408949cb9324 Mon Sep 17 00:00:00 2001 From: Noah Treuhaft Date: Mon, 14 Sep 2026 15:08:49 -0400 Subject: [PATCH 4/5] update CONTRIBUTING.md --- CONTRIBUTING.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6a9d3be17e..80ba0944fa 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -37,12 +37,14 @@ go mod edit -replace=github.com/org/repo=../repo Before any PRs are merged to main, all tests must pass. -Unit tests require Node.js. To run them, execute: +To run unit tests, execute: ``` make test-unit ``` -System tests require Python 3.3 or better. To run them, execute: +System tests require Python 3.3 or better and the +[AWS CLI](https://docs.aws.amazon.com/cli/). To run them, execute: + ``` make test-system ``` From 7aa6481825987c78235239666d011b2e3c0e6510 Mon Sep 17 00:00:00 2001 From: Noah Treuhaft Date: Mon, 14 Sep 2026 15:13:03 -0400 Subject: [PATCH 5/5] update AWS CLI URL --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 80ba0944fa..0b29f6e57e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -43,7 +43,7 @@ make test-unit ``` System tests require Python 3.3 or better and the -[AWS CLI](https://docs.aws.amazon.com/cli/). To run them, execute: +[AWS CLI](https://aws.amazon.com/cli/). To run them, execute: ``` make test-system