diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6a9d3be17e..0b29f6e57e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -37,12 +37,14 @@ go mod edit -replace=github.com/org/repo=../repo Before any PRs are merged to main, all tests must pass. -Unit tests require Node.js. To run them, execute: +To run unit tests, execute: ``` make test-unit ``` -System tests require Python 3.3 or better. To run them, execute: +System tests require Python 3.3 or better and the +[AWS CLI](https://aws.amazon.com/cli/). To run them, execute: + ``` make test-system ``` diff --git a/Makefile b/Makefile index 95142f7981..174184a94c 100644 --- a/Makefile +++ b/Makefile @@ -34,11 +34,6 @@ tidy: go mod tidy git diff --exit-code -- go.mod go.sum -bin/minio: Makefile - @curl -fLo $@ --compressed --create-dirs \ - https://dl.min.io/server/minio/release/$$(go env GOOS)-$$(go env GOARCH)/archive/minio.RELEASE.2022-05-04T07-45-27Z - @chmod +x $@ - generate: go generate ./... @@ -48,18 +43,28 @@ test-generate: generate test-unit: @go test -short ./... -test-system: build bin/minio +test-system: build bin/rustfs @ZTEST_PATH="$(CURDIR)/dist:$(CURDIR)/bin" go test . -test-boomerangs: build bin/minio +test-boomerangs: build bin/rustfs @ZTEST_PATH="$(CURDIR)/dist:$(CURDIR)/bin" go test -parallel 1 . -run TestSPQ/boomerang -v -test-run: build bin/minio +test-run: build bin/rustfs @ZTEST_PATH="$(CURDIR)/dist:$(CURDIR)/bin" go test . -v -run $(TEST) test-heavy: build @PATH="$(CURDIR)/dist:$(PATH)" go test -tags=heavy ./mdtest +bin/rustfs: Makefile + arch=$$(uname -m | sed -e s/^arm/aarch/) \ + os=$$(uname -s | sed -e s/^Darwin/macos/ | tr A-Z a-z) && \ + [ $$os != linux ] || libc=-musl && \ + url=https://github.com/rustfs/rustfs/releases/download/1.0.0-rc.5/rustfs-$${os}-$${arch}$${libc}-latest.zip && \ + echo $$url && \ + curl -Lf --create-dirs -o $@.zip $$url + unzip -oq $@.zip -d $(@D) + touch -c $@ + build: $(PEG_DEP) @mkdir -p dist @go build -ldflags='$(LDFLAGS)' -o dist ./cmd/... diff --git a/cmd/super/ztests/s3-multiple.yaml b/cmd/super/ztests/s3-multiple.yaml index 0caab26b80..e876c7984b 100644 --- a/cmd/super/ztests/s3-multiple.yaml +++ b/cmd/super/ztests/s3-multiple.yaml @@ -1,11 +1,13 @@ script: | - source minio.sh - mv *.sup data/bucket + source s3.sh + aws s3api create-bucket --bucket bucket + aws s3 cp --quiet log1.sup s3://bucket + aws s3 cp --quiet log2.sup s3://bucket super -s -c "sort ts" s3://bucket/log1.sup s3://bucket/log2.sup inputs: - - name: minio.sh - source: ../../../testdata/minio.sh + - name: s3.sh + source: ../../../testdata/s3.sh - name: log1.sup data: | {ts:2018-03-24T17:15:21.255387Z,uid:"C8Tful1TvM3Zf5x8fl"} diff --git a/cmd/super/ztests/s3-notexist.yaml b/cmd/super/ztests/s3-notexist.yaml index 6c3a2653d0..348bd2649b 100644 --- a/cmd/super/ztests/s3-notexist.yaml +++ b/cmd/super/ztests/s3-notexist.yaml @@ -1,10 +1,11 @@ script: | - source minio.sh + source s3.sh + aws s3api create-bucket --bucket bucket ! super -s s3://bucket/does/not/exist inputs: - - name: minio.sh - source: ../../../testdata/minio.sh + - name: s3.sh + source: ../../../testdata/s3.sh outputs: - name: stderr diff --git a/cmd/super/ztests/s3-parquet.yaml b/cmd/super/ztests/s3-parquet.yaml index 321f0265a4..77890d8b2c 100644 --- a/cmd/super/ztests/s3-parquet.yaml +++ b/cmd/super/ztests/s3-parquet.yaml @@ -1,11 +1,12 @@ script: | - source minio.sh - mv dns.parquet data/bucket + source s3.sh + aws s3api create-bucket --bucket bucket + aws s3 cp --quiet dns.parquet s3://bucket super -s -c "count()" s3://bucket/dns.parquet inputs: - - name: minio.sh - source: ../../../testdata/minio.sh + - name: s3.sh + source: ../../../testdata/s3.sh - name: dns.parquet source: ../../../sio/parquetio/ztests/dns.parquet diff --git a/cmd/super/ztests/s3-simple.yaml b/cmd/super/ztests/s3-simple.yaml index 1a161be574..3571303cb0 100644 --- a/cmd/super/ztests/s3-simple.yaml +++ b/cmd/super/ztests/s3-simple.yaml @@ -1,13 +1,14 @@ script: | - source minio.sh - mv babble.sup ./data/bucket + source s3.sh + aws s3api create-bucket --bucket bucket + aws s3 cp --quiet babble.sup s3://bucket super -s -c "count()" s3://bucket/babble.sup inputs: - name: babble.sup source: ../../../testdata/babble.sup - - name: minio.sh - source: ../../../testdata/minio.sh + - name: s3.sh + source: ../../../testdata/s3.sh outputs: - name: stdout diff --git a/db/ztests/s3/rm.yaml b/db/ztests/s3/rm.yaml index 8a7fcdba1e..92141afd79 100644 --- a/db/ztests/s3/rm.yaml +++ b/db/ztests/s3/rm.yaml @@ -2,7 +2,8 @@ skip: issue 2540... need to delete by key range script: | mkdir logs - source minio.sh + source s3.sh + aws s3api create-bucket --bucket bucket super db import -R logs -data s3://bucket/db babble.sup super db map -R logs -o count.bsup "count()" echo === @@ -15,8 +16,8 @@ script: | inputs: - name: babble.sup source: ../../../testdata/babble.sup - - name: minio.sh - source: ../../../testdata/minio.sh + - name: s3.sh + source: ../../../testdata/s3.sh outputs: - name: stdout diff --git a/db/ztests/s3/stat.yaml b/db/ztests/s3/stat.yaml index ca5d12669f..d5f7bcb41e 100644 --- a/db/ztests/s3/stat.yaml +++ b/db/ztests/s3/stat.yaml @@ -1,5 +1,6 @@ script: | - source minio.sh + source s3.sh + aws s3api create-bucket --bucket bucket export SUPER_DB=s3://bucket/db super db init -q super db create -q logs @@ -9,8 +10,8 @@ script: | inputs: - name: babble.sup source: ../../../testdata/babble.sup - - name: minio.sh - source: ../../../testdata/minio.sh + - name: s3.sh + source: ../../../testdata/s3.sh outputs: - name: stdout diff --git a/db/ztests/s3/zq.yaml b/db/ztests/s3/zq.yaml index fbc57e4407..2314fa23d6 100644 --- a/db/ztests/s3/zq.yaml +++ b/db/ztests/s3/zq.yaml @@ -1,5 +1,6 @@ script: | - source minio.sh + source s3.sh + aws s3api create-bucket --bucket bucket export SUPER_DB=s3://bucket/db super db init -q super db create -q logs @@ -9,8 +10,8 @@ script: | inputs: - name: babble.sup source: ../../../testdata/babble.sup - - name: minio.sh - source: ../../../testdata/minio.sh + - name: s3.sh + source: ../../../testdata/s3.sh outputs: - name: stdout diff --git a/testdata/minio.sh b/testdata/s3.sh similarity index 50% rename from testdata/minio.sh rename to testdata/s3.sh index 153dfa50ce..965833cc8f 100644 --- a/testdata/minio.sh +++ b/testdata/s3.sh @@ -1,14 +1,19 @@ #!/bin/bash -mkdir -p data/bucket +export AWS_ACCESS_KEY_ID=admin +export AWS_SECRET_ACCESS_KEY=secret -# Allocate a port. Another process could bind to it before MinIO does, +# Allocate a port. Another process could bind to it before ours does, # but that's very unlikely. port=$(python3 -c "import socket; print(socket.create_server(('localhost', 0)).getsockname()[1])") -minio server --address localhost:$port --console-address localhost:0 --quiet data & + +dir=$PWD/s3 +mkdir $dir +RUSTFS_OBS_LOG_DIRECTORY=rustfs rustfs server $dir \ + --address localhost:$port --access-key $AWS_ACCESS_KEY_ID --secret-key $AWS_SECRET_ACCESS_KEY & trap "kill -9 $!" EXIT -# Wait for MinIO to accept a connection. +# Wait for server to accept a connection. python3 <