test: Split unit, integration, and e2e tiers with just recipes

.github/workflows/ci.yml

@@ -14,14 +14,15 @@ jobs:
       - uses: actions/checkout@v7
       - uses: actions/setup-go@v7
         with: { go-version: '1.27.0' }
-      - run: go vet ./...
-      # Build-tagged files are invisible to the lint job's own GOOS, so a
-      # //go:build windows file would otherwise reach master unvetted.
-      - run: GOOS=windows go vet ./...
+      # Build-tagged files are invisible to vet and lint unless named: the
+      # e2e tests need `-tags e2e`, and a //go:build windows file needs the
+      # lint job to run under GOOS=windows, or they reach master unvetted.
+      - run: go vet -tags e2e ./...
+      - run: GOOS=windows go vet -tags e2e ./...
       - uses: golangci/golangci-lint-action@v9
-        with: { version: latest }
+        with: { version: latest, args: --build-tags=e2e }
       - uses: golangci/golangci-lint-action@v9
-        with: { version: latest }
+        with: { version: latest, args: --build-tags=e2e }
         env: { GOOS: windows }
       - name: govulncheck
         run: go run golang.org/x/vuln/cmd/govulncheck@v1.7.0 ./...
@@ -35,7 +36,7 @@ jobs:
       - uses: actions/checkout@v7
       - uses: actions/setup-go@v7
         with: { go-version: '1.27.0' }
-      - run: go test -race -count=2 -coverprofile=coverage.out ./...
+      - run: go test -race -count=2 -tags e2e -coverprofile=coverage.out ./...
         shell: bash # PowerShell splits "-coverprofile=coverage.out" and passes ".out" as a package
       - name: Coverage summary
         if: runner.os == 'Linux'
@@ -46,7 +47,7 @@ jobs:
       - uses: actions/checkout@v7
       - uses: actions/setup-go@v7
         with: { go-version: '1.26.7' } # must match the `go` directive's minor in go.mod
-      - run: go test -count=1 ./...
+      - run: go test -count=1 -tags e2e ./...
   api-compat:
     runs-on: ubuntu-latest
     steps:

.github/workflows/release.yml

@@ -13,7 +13,7 @@ jobs:
       - uses: actions/checkout@v7
       - uses: actions/setup-go@v7
         with: { go-version: '1.27.0' }
-      - run: go test -race -count=1 ./...
+      - run: go test -race -count=1 -tags e2e ./...
   release:
     needs: test
     runs-on: ubuntu-latest

AGENTS.md

@@ -25,7 +25,7 @@ phones home.
 | Language | Go ≥ 1.26 (`go.mod` minimum); CI/release pinned via workflows, `GOTOOLCHAIN=local` |
 | Dependencies | Standard library only, no CGO |
 | Module | `github.com/korya/netquality` |
-| Build/test | `go build ./...`, `go test -race ./...`, `go vet ./...`, `golangci-lint run ./...` |
+| Build/test | `go build ./...`, `go test -race -tags e2e ./...`, `go vet -tags e2e ./...`, `golangci-lint run --build-tags e2e ./...`; `just test` (unit + integration), `just test -short` (unit only), `just test-e2e` |
 | Lint config | `.golangci.yml` (standard set; `fmt.Fprint*` and `Body.Close` excluded from errcheck) |
 | CI | GitHub Actions: lint + govulncheck, tests on Linux/macOS/Windows, minimum-Go job, 6-target cross-compile, public-API compatibility gate; nightly `Live`; tag-triggered `Release` |
 | Licence | Apache-2.0; no GPL-derived code |

README.md

@@ -319,8 +319,12 @@ normalised to 1 RTT for TLS 1.3 and 2 for TLS 1.2.
 ## Testing
 
 ```
-go test ./...            # unit + end-to-end against an in-process nqserver
+just test                # unit + integration (in-process nqserver)
+just test -short         # unit only: nothing starts a listener or a process
+just test-e2e            # end-to-end tests (files tagged `//go:build e2e`)
+go test -tags e2e ./...  # everything, as CI runs it
 NQ_LIVE=1 go test -run TestLive -v .   # hits Apple and Cloudflare
+just measure-cloudflare  # a real measurement with the nq CLI (also measure-apple)
 ```
 
 [`docs/test-matrix.md`](docs/test-matrix.md) maps every feature and use case

auth_test.go

@@ -140,6 +140,7 @@ func TestBudgetExhaustedMidRunIsGraceful(t *testing.T) {
 }
 
 func TestConnectionCapDoesNotBreakRun(t *testing.T) {
+	skipIfShort(t)
 	// A cap of 8 with a client that wants up to 16 flows plus probes: the
 	// run completes (fewer flows), never errors. The listener must be wrapped
 	// before the server starts accepting.
@@ -171,6 +172,7 @@ func TestConnectionCapDoesNotBreakRun(t *testing.T) {
 // role a product backend plays in the signed-URL flow.
 func signedBackend(t *testing.T, nqURL string, key []byte, exp time.Time, sub string) *httptest.Server {
 	t.Helper()
+	skipIfShort(t)
 	sign := func(path string) string {
 		s, err := server.SignURL(key, nqURL+path, exp, sub)
 		if err != nil {

cmd/nq/e2e_test.go

@@ -1,3 +1,5 @@
+//go:build e2e
+
 package main
 
 import (
@@ -16,17 +18,14 @@ import (
 
 func startServer(t *testing.T) string {
 	t.Helper()
+	skipIfShort(t)
 	srv := httptest.NewUnstartedServer(server.Handler(server.Options{MaxClientBytes: -1}))
 	srv.EnableHTTP2 = true
 	srv.StartTLS()
 	t.Cleanup(srv.Close)
 	return srv.URL + server.ConfigPath
 }
 
-func base(url string, extra ...string) []string {
-	return append([]string{"--config-url", url, "--insecure", "--max-duration", "300ms", "--interval", "100ms", "--idle-probes", "2"}, extra...)
-}
-
 func TestJSONOutput(t *testing.T) {
 	var out, errb bytes.Buffer
 	if c := run(base(startServer(t), "--json"), &out, &errb); c != exitOK {
@@ -151,6 +150,7 @@ func TestHelpers(t *testing.T) {
 }
 
 func TestAuthTokenFlagAndEnv(t *testing.T) {
+	skipIfShort(t)
 	srv := httptest.NewUnstartedServer(server.Handler(server.Options{AuthToken: "s3cret", MaxClientBytes: -1}))
 	srv.EnableHTTP2 = true
 	srv.StartTLS()
@@ -224,12 +224,3 @@ func TestEdgeFlags(t *testing.T) {
 		t.Errorf("zero flags must mean defaults: %v idle=%+v flows=%d", err, res.Idle, res.Download.Flows)
 	}
 }
-
-func lineWith(s, prefix string) string {
-	for _, l := range strings.Split(s, "\n") {
-		if strings.HasPrefix(l, prefix) {
-			return l
-		}
-	}
-	return ""
-}

cmd/nq/helpers_test.go

@@ -0,0 +1,28 @@
+package main
+
+import (
+	"strings"
+	"testing"
+)
+
+// skipIfShort marks a test that starts a listener or a process: by this repo's definition it is
+// not a unit test, and `go test -short` skips it.
+func skipIfShort(tb testing.TB) {
+	tb.Helper()
+	if testing.Short() {
+		tb.Skip("starts a listener or process; skipped under -short")
+	}
+}
+
+func base(url string, extra ...string) []string {
+	return append([]string{"--config-url", url, "--insecure", "--max-duration", "300ms", "--interval", "100ms", "--idle-probes", "2"}, extra...)
+}
+
+func lineWith(s, prefix string) string {
+	for _, l := range strings.Split(s, "\n") {
+		if strings.HasPrefix(l, prefix) {
+			return l
+		}
+	}
+	return ""
+}

cmd/nq/idle_timeout_test.go

@@ -20,6 +20,7 @@ import (
 
 func idleStallServer(t *testing.T, h2, body bool) string {
 	t.Helper()
+	skipIfShort(t)
 	var loading atomic.Bool
 	var small atomic.Int64
 	h := server.Handler(server.Options{MaxClientBytes: -1})
@@ -83,6 +84,7 @@ func TestIdleTimeoutOutput(t *testing.T) {
 }
 
 func TestIdleTimeoutBinary(t *testing.T) {
+	skipIfShort(t)
 	bin := filepath.Join(t.TempDir(), "nq.exe")
 	buildCtx, stopBuild := context.WithTimeout(context.Background(), time.Minute)
 	defer stopBuild()

cmd/nq/probe_size_test.go

@@ -15,6 +15,7 @@ import (
 )
 
 func TestInvalidProbeOutput(t *testing.T) {
+	skipIfShort(t)
 	for _, mode := range []string{"json", "human", "events"} {
 		t.Run(mode, func(t *testing.T) {
 			var small atomic.Int64

cmd/nqserver/budget_test.go

@@ -28,6 +28,7 @@ func TestClientConcurrencyDisabledWarning(t *testing.T) {
 		{"explicit concurrency enabled budget", []string{"--client-bytes", "1048576", "--client-concurrency", "1"}, false},
 	} {
 		t.Run(tc.name, func(t *testing.T) {
+			skipIfShort(t)
 			ctx, cancel := context.WithCancel(context.Background())
 			defer cancel()
 			var out bytes.Buffer

cmd/nqserver/helpers_test.go

@@ -0,0 +1,12 @@
+package main
+
+import "testing"
+
+// skipIfShort marks a test that starts a listener or a process: by this repo's definition it is
+// not a unit test, and `go test -short` skips it.
+func skipIfShort(tb testing.TB) {
+	tb.Helper()
+	if testing.Short() {
+		tb.Skip("starts a listener or process; skipped under -short")
+	}
+}

cmd/nqserver/main_test.go

@@ -125,6 +125,7 @@ func loadLeaf(t *testing.T, certPath string) *x509.Certificate {
 // serve starts the server with args on an ephemeral port and returns its address.
 func serve(t *testing.T, args ...string) net.Addr {
 	t.Helper()
+	skipIfShort(t)
 	ctx, cancel := context.WithCancel(context.Background())
 	addrCh := make(chan net.Addr, 1)
 	done := make(chan int, 1)

e2e_test.go

@@ -1,3 +1,5 @@
+//go:build e2e
+
 package netquality
 
 import (
@@ -15,31 +17,6 @@ import (
 	"github.com/korya/netquality/server"
 )
 
-// startServer starts an in-process nqserver with the given handler wrapper
-// and TLS tweaks, returning the base URL.
-func startServer(t *testing.T, o server.Options, wrap func(http.Handler) http.Handler, tlsCfg *tls.Config, h2 bool) *httptest.Server {
-	t.Helper()
-	if o.MaxClientBytes == 0 {
-		o.MaxClientBytes = -1 // loopback moves gigabytes per run
-	}
-	h := server.Handler(o)
-	if wrap != nil {
-		h = wrap(h)
-	}
-	srv := httptest.NewUnstartedServer(h)
-	srv.EnableHTTP2 = h2
-	srv.TLS = tlsCfg
-	srv.StartTLS()
-	t.Cleanup(srv.Close)
-	return srv
-}
-
-func insecureClient() *http.Client {
-	tr := http.DefaultTransport.(*http.Transport).Clone()
-	tr.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} //nolint:gosec // test server
-	return &http.Client{Transport: tr}
-}
-
 func TestHTTP11Fallback(t *testing.T) {
 	srv := startServer(t, server.Options{}, nil, nil, false)
 	res, err := Run(context.Background(), Target{ConfigURL: srv.URL + server.ConfigPath}, Options{
@@ -121,6 +98,7 @@ func TestFlowErrorAfterIntervalsKeepsResult(t *testing.T) {
 }
 
 func TestTestEndpointHonoured(t *testing.T) {
+	skipIfShort(t)
 	// Config URLs name an unresolvable host; test_endpoint points at loopback.
 	srv := httptest.NewUnstartedServer(nil)
 	srv.EnableHTTP2 = true
@@ -375,15 +353,6 @@ func TestMaxFlowsDefaultAndOptionsDefaults(t *testing.T) {
 	}
 }
 
-func hasWarning(res *Result, substr string) bool {
-	for _, w := range res.Warnings {
-		if strings.Contains(w, substr) {
-			return true
-		}
-	}
-	return false
-}
-
 // TestTestEndpointCustomTLSDialer (DISC-9): a custom TLS dialer cannot honour
 // test_endpoint, and the run says so instead of silently ignoring it.
 func TestTestEndpointCustomTLSDialer(t *testing.T) {

helpers_test.go

@@ -0,0 +1,55 @@
+package netquality
+
+import (
+	"crypto/tls"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/korya/netquality/server"
+)
+
+// skipIfShort marks a test that starts a listener or a process: by this repo's definition it is
+// not a unit test, and `go test -short` skips it.
+func skipIfShort(tb testing.TB) {
+	tb.Helper()
+	if testing.Short() {
+		tb.Skip("starts a listener or process; skipped under -short")
+	}
+}
+
+// startServer starts an in-process nqserver with the given handler wrapper
+// and TLS tweaks, returning the base URL.
+func startServer(t *testing.T, o server.Options, wrap func(http.Handler) http.Handler, tlsCfg *tls.Config, h2 bool) *httptest.Server {
+	t.Helper()
+	skipIfShort(t)
+	if o.MaxClientBytes == 0 {
+		o.MaxClientBytes = -1 // loopback moves gigabytes per run
+	}
+	h := server.Handler(o)
+	if wrap != nil {
+		h = wrap(h)
+	}
+	srv := httptest.NewUnstartedServer(h)
+	srv.EnableHTTP2 = h2
+	srv.TLS = tlsCfg
+	srv.StartTLS()
+	t.Cleanup(srv.Close)
+	return srv
+}
+
+func insecureClient() *http.Client {
+	tr := http.DefaultTransport.(*http.Transport).Clone()
+	tr.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} //nolint:gosec // test server
+	return &http.Client{Transport: tr}
+}
+
+func hasWarning(res *Result, substr string) bool {
+	for _, w := range res.Warnings {
+		if strings.Contains(w, substr) {
+			return true
+		}
+	}
+	return false
+}

justfile

@@ -1,3 +1,27 @@
+_default:
+    just --list
+
+# Run unit and integration tests (extra args go to `go test`; `-short` keeps only unit tests).
+test *args:
+    go test -count=1 {{ args }} ./...
+
+# Run only the end-to-end tests: every Test* in files tagged `//go:build e2e` (extra args go to `go test`).
+test-e2e *args:
+    #!/usr/bin/env sh
+    set -eu
+    for f in $(git ls-files '*_test.go' | xargs grep -l '^//go:build e2e$'); do
+        names=$(grep -o '^func Test[A-Za-z0-9_]*' "$f" | sed 's/^func //' | paste -sd '|' -)
+        go test -count=1 -tags e2e -run "^($names)\$" {{ args }} "./$(dirname "$f")"
+    done
+
+# Run a real measurement against Cloudflare with the nq CLI (extra args are nq flags, e.g. --json).
+measure-cloudflare *args:
+    go run ./cmd/nq --target cloudflare {{ args }}
+
+# Run a real measurement against Apple with the nq CLI (extra args are nq flags, e.g. --json).
+measure-apple *args:
+    go run ./cmd/nq --target apple {{ args }}
+
 # Run every package benchmark without running the regular test suite.
 bench:
     go test -run '^$' -bench . -benchmem ./...

probe_size_test.go

@@ -51,6 +51,7 @@ func (b *trackedProbeBody) Close() error {
 }
 
 func TestProbeResponseSize(t *testing.T) {
+	skipIfShort(t)
 	for _, h2 := range []bool{false, true} {
 		for _, tc := range []struct {
 			name     string
@@ -140,6 +141,7 @@ func TestProbeResponseSize(t *testing.T) {
 }
 
 func TestProbeRejectionPreservesLoad(t *testing.T) {
+	skipIfShort(t)
 	for _, mode := range []string{"declared", "streamed", "deadline"} {
 		t.Run(mode, func(t *testing.T) {
 			more := make(chan struct{})