diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f627e49469..d5d436aaca 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -8,6 +8,7 @@ stages: variables: GIT_SUBMODULE_STRATEGY: recursive + GITHUB_RELEASES_MIRROR: "https://depot-read-api-github-releases.us1.ddbuild.io/internal/mirror/github-releases" # Only clone submodules required by default test jobs # libddwaf-rust is needed because the root Cargo workspace includes # appsec/helper-rust, which depends on it (see 3f893e263) diff --git a/.gitlab/dockerhub-login.sh b/.gitlab/dockerhub-login.sh index 357f73d69b..fcb404f4e3 100755 --- a/.gitlab/dockerhub-login.sh +++ b/.gitlab/dockerhub-login.sh @@ -3,6 +3,7 @@ set -e export VAULT_VERSION="1.20.0" +: "${GITHUB_RELEASES_MIRROR:?GITHUB_RELEASES_MIRROR must be set}" echo "=== Setting up Docker Hub authentication ===" @@ -27,7 +28,7 @@ if ! command -v jq > /dev/null 2>&1; then jq_path="/tmp/jq" - if ! curl -L --fail "https://github.com/jqlang/jq/releases/latest/download/jq-linux-${vault_arch}" \ + if ! curl -L --fail "${GITHUB_RELEASES_MIRROR}/jqlang/jq/releases/download/latest/jq-linux-${vault_arch}" \ --output "${jq_path}"; then echo "Warning: Failed to download jq. Skipping Docker Hub authentication." >&2 exit 0 diff --git a/.gitlab/generate-ci-images.php b/.gitlab/generate-ci-images.php index 027449b618..360cf03506 100644 --- a/.gitlab/generate-ci-images.php +++ b/.gitlab/generate-ci-images.php @@ -256,7 +256,7 @@ function parse_compose(string $path, array $env): array Write-Host "Downloading docker-compose..." [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 $dockerCompose = "$PWD\docker-compose.exe" - Start-BitsTransfer -Source "https://github.com/docker/compose/releases/download/v2.36.0/docker-compose-windows-x86_64.exe" -Destination $dockerCompose + Start-BitsTransfer -Source "$env:GITHUB_RELEASES_MIRROR/docker/compose/releases/download/v2.36.0/docker-compose-windows-x86_64.exe" -Destination $dockerCompose # Scope Docker auth to this job: CI Identities for registry.ddbuild.io only, # no ECR catch-all that would fail without AWS_DEFAULT_REGION. Docker calls diff --git a/.gitlab/generate-package.php b/.gitlab/generate-package.php index 2d860b2bdd..22d24d0320 100644 --- a/.gitlab/generate-package.php +++ b/.gitlab/generate-package.php @@ -897,7 +897,7 @@ function package_extension_jobs(array $platform, string $kind = "all"): array - .gitlab/run-with-retryable-download.sh apt-get update - .gitlab/run-with-retryable-download.sh apt install -y php git make curl - - curl -L --fail https://github.com/docker/compose/releases/download/v2.36.0/${DOCKER_COMPOSE_DOWNLOAD_NAME} -o /usr/local/bin/docker-compose + - curl -L --fail ${GITHUB_RELEASES_MIRROR}/docker/compose/releases/download/v2.36.0/${DOCKER_COMPOSE_DOWNLOAD_NAME} -o /usr/local/bin/docker-compose - chmod +x /usr/local/bin/docker-compose - mv packages/* . - docker network create randomized_tests_baseservices @@ -1094,7 +1094,7 @@ function package_extension_jobs(array $platform, string $kind = "all"): array - apt-get update - apt install -y make curl - - curl -L --fail https://github.com/docker/compose/releases/download/v2.36.0/docker-compose-linux-x86_64 -o /usr/local/bin/docker-compose + - curl -L --fail ${GITHUB_RELEASES_MIRROR}/docker/compose/releases/download/v2.36.0/docker-compose-linux-x86_64 -o /usr/local/bin/docker-compose - chmod +x /usr/local/bin/docker-compose - mkdir build - mv packages build diff --git a/.gitlab/upload-code-coverage-to-datadog.sh b/.gitlab/upload-code-coverage-to-datadog.sh index 5e5d0f738d..af899a02cc 100755 --- a/.gitlab/upload-code-coverage-to-datadog.sh +++ b/.gitlab/upload-code-coverage-to-datadog.sh @@ -8,6 +8,7 @@ if [[ $# -eq 0 ]]; then fi : "${CI_PROJECT_DIR:?CI_PROJECT_DIR must be set}" +: "${GITHUB_RELEASES_MIRROR:?GITHUB_RELEASES_MIRROR must be set}" export DD_SITE="${DD_SITE:-datadoghq.com}" @@ -81,7 +82,7 @@ fi if [[ ! -x "${datadog_ci_path}" ]]; then if ! curl -L --fail \ - "https://github.com/DataDog/datadog-ci/releases/download/${datadog_ci_version}/datadog-ci_linux-${datadog_ci_arch}" \ + "${GITHUB_RELEASES_MIRROR}/DataDog/datadog-ci/releases/download/${datadog_ci_version}/datadog-ci_linux-${datadog_ci_arch}" \ --output "${datadog_ci_path}"; then echo "ERROR: failed to download datadog-ci; exiting 75 so GitLab auto-retries" >&2 exit 75 diff --git a/.gitlab/upload-junit-to-datadog.sh b/.gitlab/upload-junit-to-datadog.sh index 00067683ad..ef63980869 100755 --- a/.gitlab/upload-junit-to-datadog.sh +++ b/.gitlab/upload-junit-to-datadog.sh @@ -7,6 +7,7 @@ export DD_ENV="ci" export DD_SERVICE="${DD_SERVICE:-dd-trace-php-tests}" export VAULT_SECRET_PATH="kv/k8s/gitlab-runner/dd-trace-php/datadoghq-api-key" export VAULT_VERSION="1.20.0" +: "${GITHUB_RELEASES_MIRROR:?GITHUB_RELEASES_MIRROR must be set}" # Parse arguments for tags (e.g., component:tracer test.type:unit) TAGS="${*}" @@ -162,7 +163,7 @@ else echo "Downloading datadog-ci standalone binary..." datadog_ci_path="/tmp/datadog-ci" - datadog_ci_url="https://github.com/DataDog/datadog-ci/releases/latest/download/datadog-ci_linux-${datadog_ci_arch}" + datadog_ci_url="${GITHUB_RELEASES_MIRROR}/DataDog/datadog-ci/releases/download/latest/datadog-ci_linux-${datadog_ci_arch}" if ! curl -L --fail "${datadog_ci_url}" --output "${datadog_ci_path}"; then echo "Warning: Failed to download datadog-ci binary. Skipping JUnit upload." >&2 diff --git a/appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/docker/AppSecContainer.groovy b/appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/docker/AppSecContainer.groovy index 4f0f2c3866..cef7f9180e 100644 --- a/appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/docker/AppSecContainer.groovy +++ b/appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/docker/AppSecContainer.groovy @@ -95,6 +95,10 @@ class AppSecContainer> extends GenericContain withEnv 'DD_TRACE_GIT_METADATA_ENABLED', '0' withEnv 'DD_EXPERIMENTAL_PROPAGATE_PROCESS_TAGS_ENABLED', '0' withEnv 'DD_INSTRUMENTATION_TELEMETRY_ENABLED', '1' + String githubReleasesMirror = System.getenv('GITHUB_RELEASES_MIRROR') + if (githubReleasesMirror) { + withEnv 'GITHUB_RELEASES_MIRROR', githubReleasesMirror + } // very verbose: withEnv '_DD_DEBUG_SIDECAR_LOG_METHOD', 'file:///tmp/logs/sidecar.log' withEnv 'DD_TELEMETRY_HEARTBEAT_INTERVAL', '10' diff --git a/dockerfiles/verify_packages/Makefile b/dockerfiles/verify_packages/Makefile index 35a6a5966a..09e39aa4b1 100644 --- a/dockerfiles/verify_packages/Makefile +++ b/dockerfiles/verify_packages/Makefile @@ -9,6 +9,7 @@ INSTALLER_DOCKER_RUN = docker run --rm -i \ --env-file $(shell pwd)/.env \ $(INSTALLER_DOCKER_ARGS) \ -e CIRCLECI \ + -e GITHUB_RELEASES_MIRROR \ --workdir=/app .PHONY: $(CENTOS6_PHP_VERSIONS) diff --git a/dockerfiles/verify_packages/installer/utils.sh b/dockerfiles/verify_packages/installer/utils.sh index feb77988d0..8042120c19 100755 --- a/dockerfiles/verify_packages/installer/utils.sh +++ b/dockerfiles/verify_packages/installer/utils.sh @@ -2,6 +2,8 @@ # Common functions used to test the installation process +github_releases_base_url="${GITHUB_RELEASES_MIRROR:-https://github.com}" + assert_file_contains() { output=$(cat ${1}) if [ -z "${output##*$2*}" ]; then @@ -204,7 +206,7 @@ assert_world_readable_tree() { install_legacy_ddtrace() ( version=$1 curl -L --output "/tmp/legacy-${version}.tar.gz" \ - "https://github.com/DataDog/dd-trace-php/releases/download/${version}/datadog-php-tracer-${version}.x86_64.tar.gz" + "${github_releases_base_url}/DataDog/dd-trace-php/releases/download/${version}/datadog-php-tracer-${version}.x86_64.tar.gz" tar -xf "/tmp/legacy-${version}.tar.gz" -C / /opt/datadog-php/bin/post-install.sh ) @@ -235,13 +237,12 @@ fetch_setup_for_version() ( mkdir -vp "${destdir?}" cd "${destdir}" - curl -OL https://github.com/DataDog/dd-trace-php/releases/download/${version}/datadog-setup.php + curl -OL "${github_releases_base_url}/DataDog/dd-trace-php/releases/download/${version}/datadog-setup.php" cd - ) run_released_installer() ( - unset DD_TEST_INSTALLER_REPO - php "$@" + DD_TEST_INSTALLER_REPO="${github_releases_base_url}/DataDog/dd-trace-php" php "$@" ) parse_appsec_version() { diff --git a/tests/Frameworks/Roadrunner/Version_2/run.sh b/tests/Frameworks/Roadrunner/Version_2/run.sh index 86b5b34308..65b5b1413d 100755 --- a/tests/Frameworks/Roadrunner/Version_2/run.sh +++ b/tests/Frameworks/Roadrunner/Version_2/run.sh @@ -3,6 +3,8 @@ set -x set -o pipefail +github_releases_base_url="${GITHUB_RELEASES_MIRROR:-https://github.com}" + cd /var/www composer install --no-dev @@ -15,7 +17,7 @@ if [[ ! -f rr ]]; then ARCH="amd64" fi - curl -Lf https://github.com/roadrunner-server/roadrunner/releases/download/v2.12.3/roadrunner-2.12.3-linux-$ARCH.tar.gz | \ + curl -Lf "${github_releases_base_url}/roadrunner-server/roadrunner/releases/download/v2.12.3/roadrunner-2.12.3-linux-${ARCH}.tar.gz" | \ tar -xzf - --strip-components=1 roadrunner-2.12.3-linux-$ARCH/rr fi diff --git a/tests/Sapi/Roadrunner/RoadrunnerServer.php b/tests/Sapi/Roadrunner/RoadrunnerServer.php index e113fff07d..efa8ad241a 100644 --- a/tests/Sapi/Roadrunner/RoadrunnerServer.php +++ b/tests/Sapi/Roadrunner/RoadrunnerServer.php @@ -101,9 +101,14 @@ public function __construct($version, $workerFile, $host, $port, array $envs = [ private static function downloadRoadrunner($target, $version) { - // phpcs:disable Generic.Files.LineLength.TooLong - exec("curl -L https://github.com/roadrunner-server/roadrunner/releases/download/v$version/roadrunner-$version-linux-$target.tar.gz | tar xz -O -f - roadrunner-$version-linux-$target/rr > '" . __DIR__ . "/rr-$version-$target'; chmod +x '" . __DIR__ . "/rr-$version-$target'"); - // phpcs:enable Generic.Files.LineLength.TooLong + $baseUrl = getenv('GITHUB_RELEASES_MIRROR') ?: 'https://github.com'; + $url = "$baseUrl/roadrunner-server/roadrunner/releases/download/v$version/" + . "roadrunner-$version-linux-$target.tar.gz"; + $binary = __DIR__ . "/rr-$version-$target"; + + exec("curl -L " . escapeshellarg($url) + . " | tar xz -O -f - roadrunner-$version-linux-$target/rr > " . escapeshellarg($binary) + . "; chmod +x " . escapeshellarg($binary)); } public function start()