From 8c03217679ec3711d681708ab750acec0540a17d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Antonio=20Fern=C3=A1ndez=20de=20Alba?= Date: Fri, 28 Aug 2026 09:58:42 +0200 Subject: [PATCH 01/99] fix(test-optimization): cache settings request in the filesystem cache (#9989) --- .../exporters/ci-visibility-exporter.js | 259 ++++- .../exporters/settings-cache-key.js | 42 + .../src/ci-visibility/requests/fs-cache.js | 75 +- .../requests/get-library-configuration.js | 3 - .../exporters/ci-visibility-exporter.spec.js | 107 +++ .../ci-visibility-settings-fs-cache.spec.js | 885 ++++++++++++++++++ 6 files changed, 1314 insertions(+), 57 deletions(-) create mode 100644 packages/dd-trace/src/ci-visibility/exporters/settings-cache-key.js create mode 100644 packages/dd-trace/test/ci-visibility/exporters/ci-visibility-settings-fs-cache.spec.js diff --git a/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js b/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js index 30950ea4ed8..7f1a7ed5dd2 100644 --- a/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js +++ b/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js @@ -1,17 +1,20 @@ 'use strict' +const fs = require('node:fs') const { hostname: getHostname } = require('node:os') const URL = require('url').URL const { version: tracerVersion } = require('../../../../../package.json') const { EMPTY_EFD_RETRY_POLICY, createEfdRetryPolicy } = require('../efd-retry-policy') const { getLibraryConfiguration: getLibraryConfigurationRequest } = require('../requests/get-library-configuration') +const { getCachePath, withCache, writeToCache } = require('../requests/fs-cache') const { getSkippableSuites: getSkippableSuitesRequest } = require('../intelligent-test-runner/get-skippable-suites') const { getKnownTests: getKnownTestsRequest } = require('../early-flake-detection/get-known-tests') const { getTestManagementTests: getTestManagementTestsRequest } = require('../test-management/get-test-management-tests') const { writeSettingsToCache } = require('../test-optimization-cache') const { CACHE_MISS, TestOptimizationHttpCache } = require('../test-optimization-http-cache') +const { MAX_RETRIES } = require('../test-optimization-http-cache-schema') const { incrementCountMetric, TELEMETRY_EVENTS_ENQUEUED_FOR_SERIALIZATION } = require('../telemetry') const { uploadCoverageReport: uploadCoverageReportRequest } = require('../requests/upload-coverage-report') const { uploadTestScreenshot: uploadTestScreenshotRequest } = require('../requests/upload-test-screenshot') @@ -28,9 +31,24 @@ const { FINAL_FLUSH_TIMEOUT, } = require('../final-flush') const { sendGitMetadata: sendGitMetadataRequest } = require('./git/git_metadata') +const buildSettingsCacheKey = require('./settings-cache-key') const hostname = getHostname() const EMPTY_SETTINGS = Object.freeze({}) +const SETTINGS_CACHE_TTL_MS = 5 * 60 * 1000 // 5 minutes +const SETTINGS_BOOLEAN_FIELDS = Object.freeze([ + 'isCodeCoverageEnabled', + 'isSuitesSkippingEnabled', + 'isItrEnabled', + 'requireGit', + 'isEarlyFlakeDetectionEnabled', + 'isFlakyTestRetriesEnabled', + 'isDiEnabled', + 'isKnownTestsEnabled', + 'isTestManagementEnabled', + 'isImpactedTestsEnabled', + 'isCoverageReportUploadEnabled', +]) /** * Test session identity sent with every request. Fields are optional because the CI provider, @@ -62,6 +80,80 @@ function getIsTestSessionTrace (trace) { ) } +/** + * Checks whether a value is a non-negative safe integer. + * + * @param {unknown} value - Candidate integer. + * @returns {value is number} + */ +function isNonNegativeSafeInteger (value) { + return Number.isSafeInteger(value) && value >= 0 +} + +/** + * Checks whether a value is a retry count accepted by the settings parser. + * + * @param {unknown} value - Candidate retry count. + * @returns {value is number} + */ +function isValidRetryCount (value) { + return isNonNegativeSafeInteger(value) && value <= MAX_RETRIES +} + +/** + * Checks whether a cached EFD retry policy has the complete parsed shape. + * + * @param {unknown} retryPolicy - Candidate retry policy. + * @returns {boolean} + */ +function isValidCachedEfdRetryPolicy (retryPolicy) { + if (retryPolicy === null || typeof retryPolicy !== 'object' || Array.isArray(retryPolicy)) return false + if (!isValidRetryCount(retryPolicy.schedulingRetryCount)) return false + if (!Array.isArray(retryPolicy.durationRetryCounts) || + retryPolicy.durationRetryCounts.length !== EMPTY_EFD_RETRY_POLICY.durationRetryCounts.length) { + return false + } + + let schedulingRetryCount = 0 + for (let index = 0; index < EMPTY_EFD_RETRY_POLICY.durationRetryCounts.length; index++) { + const durationRetryCount = retryPolicy.durationRetryCounts[index] + if (durationRetryCount === null || typeof durationRetryCount !== 'object' || + Array.isArray(durationRetryCount) || + durationRetryCount.durationLimitMs !== + EMPTY_EFD_RETRY_POLICY.durationRetryCounts[index].durationLimitMs || + !isValidRetryCount(durationRetryCount.retryCount)) { + return false + } + if (durationRetryCount.retryCount > schedulingRetryCount) { + schedulingRetryCount = durationRetryCount.retryCount + } + } + return retryPolicy.schedulingRetryCount === schedulingRetryCount +} + +/** + * Checks whether a filesystem cache value has the complete shape produced by + * parseLibraryConfigurationResponse. + * + * @param {unknown} settings - Cached settings value. + * @returns {settings is Record} + */ +function isValidCachedSettings (settings) { + if (settings === null || typeof settings !== 'object' || Array.isArray(settings)) return false + for (const field of SETTINGS_BOOLEAN_FIELDS) { + if (typeof settings[field] !== 'boolean') return false + } + if (settings.isDiEnabled && !settings.isFlakyTestRetriesEnabled) return false + if (settings.isEarlyFlakeDetectionEnabled && !settings.isKnownTestsEnabled) return false + if (!isValidCachedEfdRetryPolicy(settings.earlyFlakeDetectionRetryPolicy)) return false + if (!isNonNegativeSafeInteger(settings.earlyFlakeDetectionFaultyThreshold) || + settings.earlyFlakeDetectionFaultyThreshold > 100) { + return false + } + return settings.testManagementAttemptToFixRetries === undefined || + isValidRetryCount(settings.testManagementAttemptToFixRetries) +} + const GIT_UPLOAD_TIMEOUT = 60_000 // 60 seconds const CAN_USE_CI_VIS_PROTOCOL_TIMEOUT = GIT_UPLOAD_TIMEOUT const MAX_COVERAGE_REPORT_FLAGS = 32 @@ -100,6 +192,7 @@ class CiVisibilityExporter extends BufferingExporter { #pendingScreenshotUploads = new Set() #screenshotFlushWaiters = new Set() #deferredTestSuiteSpans = new Map() + #gitUploadTimeoutId constructor (config, options = {}) { super(config) @@ -126,11 +219,6 @@ class CiVisibilityExporter extends BufferingExporter { this._isTestFailureScreenshotsEnabled = Boolean(config?.testOptimization?.DD_TEST_FAILURE_SCREENSHOTS_ENABLED) - const gitUploadTimeoutId = setTimeout(() => { - this._resolveGit(new Error('Timeout while uploading git metadata')) - }, GIT_UPLOAD_TIMEOUT) - gitUploadTimeoutId.unref?.() - const canUseCiVisProtocolTimeoutId = setTimeout(() => { this._resolveCanUseCiVisProtocol(false) }, CAN_USE_CI_VIS_PROTOCOL_TIMEOUT) @@ -138,7 +226,8 @@ class CiVisibilityExporter extends BufferingExporter { this._gitUploadPromise = new Promise(resolve => { this._resolveGit = (err) => { - clearTimeout(gitUploadTimeoutId) + clearTimeout(this.#gitUploadTimeoutId) + this.#gitUploadTimeoutId = null resolve(err) } }) @@ -270,51 +359,142 @@ class CiVisibilityExporter extends BufferingExporter { const cachedLibraryConfig = this._testOptimizationHttpCache.readSettings() if (cachedLibraryConfig !== CACHE_MISS) { log.debug('Test Optimization HTTP cache settings found, skipping settings request') - writeSettingsToCache(cachedLibraryConfig) - this._libraryConfig = this.filterConfiguration(cachedLibraryConfig) - const canUseCachedSkippableSuites = !this.shouldRequestSkippableSuites() || - this._testOptimizationHttpCache.hasValidSkippableSuites({ - testLevel: configuration.testLevel, - isCoverageReportUploadEnabled: configuration.isCoverageReportUploadEnabled, - }) - if (this._libraryConfig.requireGit && !canUseCachedSkippableSuites) { - this.sendGitMetadata(repositoryUrl) - } else { - this._resolveGit() - } - return callback(null, this._libraryConfig) + return this._applyCachedSettings(cachedLibraryConfig, configuration, repositoryUrl, false, callback) } if (this._isTestOptimizationCacheOnly) { return callback(this._getCacheOnlyError('settings'), {}) } - this.sendGitMetadata(repositoryUrl) - getLibraryConfigurationRequest(configuration, (err, libraryConfig) => { + // The settings request has a two-phase shape: when the backend returns + // `require_git`, we upload git metadata and re-request settings. Only the + // final (post-git-upload) configuration is cached, so the cross-process + // filesystem cache never serves a pre-git-upload response that would skip + // the git upload in other processes. + // + // The live-fetch and cache-hit paths resolve the git upload promise + // differently, so they must not share a single apply path: + // - Live fetch: `sendGitMetadata` is started in `_fetchLibraryConfigurationFromBackend` + // and resolves `_gitUploadPromise` itself (with the upload result). We must NOT call + // `_resolveGit()` here, or we would race the in-flight upload and mask its error. + // - Filesystem cache hit: no upload was started in this process, so `_applyCachedSettings` + // resolves the git promise locally (starting an upload only if the cached config still + // requires git and we lack valid cached skippable suites). + let liveFetchStarted = false + const fsCacheKey = buildSettingsCacheKey(configuration) + const fetchSettings = (activeCacheKey, done) => { + liveFetchStarted = true + this._fetchLibraryConfigurationFromBackend(configuration, repositoryUrl, activeCacheKey, done) + } + const applySettings = (err, libraryConfig) => { /** * **Important**: this._libraryConfig remains empty in testing frameworks * where the tests run in a subprocess, like Jest, * because `getLibraryConfiguration` is called only once in the main process. */ - this._libraryConfig = this.filterConfiguration(libraryConfig) - if (err) { - callback(err, {}) - } else if (libraryConfig?.requireGit) { - // If the backend requires git, we'll wait for the upload to finish and request settings again - this._gitUploadPromise.then(gitUploadError => { - if (gitUploadError) { - return callback(gitUploadError, {}) + return callback(err, {}) + } + if (liveFetchStarted) { + // Live fetch: the git upload was already started and resolves + // `_gitUploadPromise` itself. Do not call `_resolveGit()`. + writeSettingsToCache(libraryConfig) + this._libraryConfig = this.filterConfiguration(libraryConfig) + return callback(null, this._libraryConfig) + } + // Filesystem cache hit: no git upload was started in this process. + if (!isValidCachedSettings(libraryConfig)) { + // A syntactically valid cache file with an invalid settings payload is corrupt. + // Settings never writes such a value, so this is external corruption. Remove the + // file and fall back to the backend so we never serve a garbage config that would + // crash filterConfiguration or silently disable every feature. + try { + fs.unlinkSync(getCachePath(fsCacheKey)) + } catch (err) { + if (err.code !== 'ENOENT') { + return fetchSettings(null, applySettings) } - getLibraryConfigurationRequest(configuration, (err, finalLibraryConfig) => { - this._libraryConfig = this.filterConfiguration(finalLibraryConfig) - callback(err, this._libraryConfig) - }) - }) - } else { - callback(null, this._libraryConfig) + } + liveFetchStarted = false + return withCache(fsCacheKey, fetchSettings, applySettings, SETTINGS_CACHE_TTL_MS) } + this._applyCachedSettings(libraryConfig, configuration, repositoryUrl, true, callback) + } + withCache(fsCacheKey, fetchSettings, applySettings, SETTINGS_CACHE_TTL_MS) + }) + } + + /** + * Applies a resolved (cached or freshly fetched) library configuration: writes it to the + * shared settings cache, filters it through local kill switches, and resolves the git + * upload promise according to whether git metadata still needs to be uploaded. + * + * @param {Record} settings - Resolved library configuration. + * @param {object} configuration - Request configuration used to evaluate cached skippable suites. + * @param {string} repositoryUrl - Repository URL for git metadata upload. + * @param {boolean} isFilesystemCache - Whether settings came from the cross-process cache. + * @param {Function} callback - Completion callback. + * @returns {void} + */ + _applyCachedSettings (settings, configuration, repositoryUrl, isFilesystemCache, callback) { + writeSettingsToCache(settings) + this._libraryConfig = this.filterConfiguration(settings) + const canUseCachedSkippableSuites = !this.shouldRequestSkippableSuites() || + this._testOptimizationHttpCache.hasValidSkippableSuites({ + testLevel: configuration.testLevel, + isCoverageReportUploadEnabled: configuration.isCoverageReportUploadEnabled, }) + if (!canUseCachedSkippableSuites && (this._libraryConfig.requireGit || isFilesystemCache)) { + this.sendGitMetadata(repositoryUrl) + } else { + this._resolveGit() + } + callback(null, this._libraryConfig) + } + + /** + * Fetches library configuration from the backend, performing the two-phase + * `require_git` re-request, and writes only the final configuration to the + * filesystem cache when this process owns the cache lock. + * + * @param {object} configuration - Request configuration for the settings endpoint. + * @param {string} repositoryUrl - Repository URL for git metadata upload. + * @param {string|null} cacheKey - Filesystem cache key when this process owns the lock, null otherwise. + * @param {Function} done - Completion callback. + * @returns {void} + */ + _fetchLibraryConfigurationFromBackend (configuration, repositoryUrl, cacheKey, done) { + this.sendGitMetadata(repositoryUrl) + getLibraryConfigurationRequest(configuration, (err, libraryConfig) => { + // Mirror the original live path: keep the phase-1 config on `_libraryConfig` even + // before the git upload resolves, so `shouldRequestSkippableSuites()` and the + // skippable path's `_gitUploadPromise` await behave identically to the uncached flow. + this._libraryConfig = this.filterConfiguration(libraryConfig) + if (err) { + return done(err, libraryConfig) + } + if (libraryConfig?.requireGit) { + // If the backend requires git, wait for the upload to finish and request settings again + this._gitUploadPromise.then(gitUploadError => { + if (gitUploadError) { + return done(gitUploadError, libraryConfig) + } + getLibraryConfigurationRequest(configuration, (finalErr, finalLibraryConfig) => { + if (finalErr) { + // Match the original live path: reset `_libraryConfig` from the final + // (failed) response so stale phase-1 feature flags don't stay installed. + // On error `finalLibraryConfig` is undefined, so this resolves to empty settings. + this._libraryConfig = this.filterConfiguration(finalLibraryConfig) + return done(finalErr, finalLibraryConfig) + } + writeToCache(cacheKey, finalLibraryConfig) + done(null, finalLibraryConfig) + }) + }) + } else { + writeToCache(cacheKey, libraryConfig) + done(null, libraryConfig) + } }) } @@ -381,6 +561,13 @@ class CiVisibilityExporter extends BufferingExporter { this._resolveGit() return } + if (this.#gitUploadTimeoutId === null) return + if (this.#gitUploadTimeoutId === undefined) { + this.#gitUploadTimeoutId = setTimeout(() => { + this._resolveGit(new Error('Timeout while uploading git metadata')) + }, GIT_UPLOAD_TIMEOUT) + this.#gitUploadTimeoutId.unref?.() + } this._canUseCiVisProtocolPromise.then((canUseCiVisProtocol) => { if (!canUseCiVisProtocol) { return diff --git a/packages/dd-trace/src/ci-visibility/exporters/settings-cache-key.js b/packages/dd-trace/src/ci-visibility/exporters/settings-cache-key.js new file mode 100644 index 00000000000..a8c3208591f --- /dev/null +++ b/packages/dd-trace/src/ci-visibility/exporters/settings-cache-key.js @@ -0,0 +1,42 @@ +'use strict' + +const { version: tracerVersion } = require('../../../../../package.json') +const { buildCacheKey } = require('../requests/fs-cache') +const getConfig = require('../../config') + +/** + * Builds the cross-process filesystem cache key for the settings request from the + * request configuration. The key also isolates the tracer version and local flags + * applied while parsing the response. + * + * @param {object} configuration - Request configuration for the settings endpoint. + * @returns {string} + */ +function buildSettingsCacheKey (configuration) { + const config = getConfig() + const { testOptimization } = config + return buildCacheKey('settings', [ + tracerVersion, + configuration.url?.href, + configuration.isEvpProxy, + configuration.evpProxyPrefix, + configuration.sha, + configuration.service, + configuration.env, + configuration.repositoryUrl, + configuration.branch, + configuration.tag, + configuration.testLevel, + configuration.osPlatform, + configuration.osVersion, + configuration.osArchitecture, + configuration.runtimeName, + configuration.runtimeVersion, + configuration.custom, + testOptimization.DD_CIVISIBILITY_DANGEROUSLY_FORCE_COVERAGE, + testOptimization.DD_CIVISIBILITY_DANGEROUSLY_FORCE_TEST_SKIPPING, + testOptimization.DD_CIVISIBILITY_CODE_COVERAGE_REPORT_UPLOAD_ENABLED, + ]) +} + +module.exports = buildSettingsCacheKey diff --git a/packages/dd-trace/src/ci-visibility/requests/fs-cache.js b/packages/dd-trace/src/ci-visibility/requests/fs-cache.js index d7b2c9d46ee..1030e793ff5 100644 --- a/packages/dd-trace/src/ci-visibility/requests/fs-cache.js +++ b/packages/dd-trace/src/ci-visibility/requests/fs-cache.js @@ -50,13 +50,25 @@ function getLockPath (cacheKey) { return path.join(tmpdir(), `dd-${cacheKey}.lock`) } +/** + * Checks whether a filesystem cache timestamp can be safely compared with the current time. + * + * @param {unknown} timestamp - Timestamp read from a cache or lock file. + * @param {number} now - Current Unix timestamp in milliseconds. + * @returns {timestamp is number} + */ +function isValidTimestamp (timestamp, now) { + return Number.isSafeInteger(timestamp) && timestamp >= 0 && timestamp <= now +} + /** * Attempts to read cached data from the filesystem. * * @param {string} cacheKey + * @param {number} [cacheTtlMs] - Maximum cache age in milliseconds. * @returns {{ data: unknown } | undefined} */ -function readFromCache (cacheKey) { +function readFromCache (cacheKey, cacheTtlMs = CACHE_TTL_MS) { const cachePath = getCachePath(cacheKey) try { const raw = fs.readFileSync(cachePath, 'utf8') @@ -66,8 +78,14 @@ function readFromCache (cacheKey) { return } const { timestamp, data } = parsed - if (Date.now() - timestamp > CACHE_TTL_MS) { - log.debug('%s cache expired (age: %d ms)', cacheKey, Date.now() - timestamp) + const now = Date.now() + if (!isValidTimestamp(timestamp, now)) { + log.debug('%s cache file has an invalid timestamp, ignoring', cacheKey) + return + } + const age = now - timestamp + if (age > cacheTtlMs) { + log.debug('%s cache expired (age: %d ms)', cacheKey, age) return } log.debug('%s cache hit', cacheKey) @@ -101,7 +119,7 @@ function writeToCache (cacheKey, data) { * Attempts to acquire an exclusive lock using O_CREAT|O_EXCL. * * @param {string} cacheKey - * @returns {boolean} + * @returns {boolean|undefined} */ function tryAcquireLock (cacheKey) { const lockPath = getLockPath(cacheKey) @@ -110,8 +128,9 @@ function tryAcquireLock (cacheKey) { fs.writeSync(fd, String(Date.now())) fs.closeSync(fd) return true - } catch { - return false + } catch (err) { + if (err.code === 'EEXIST') return false + log.debug('%s lock cannot be created, bypassing cache: %s', cacheKey, err.message) } } @@ -119,9 +138,15 @@ function tryAcquireLock (cacheKey) { * Removes the lock file. * * @param {string} cacheKey + * @returns {boolean} */ function releaseLock (cacheKey) { - try { fs.unlinkSync(getLockPath(cacheKey)) } catch { /* ignore */ } + try { + fs.unlinkSync(getLockPath(cacheKey)) + return true + } catch (err) { + return err.code === 'ENOENT' + } } /** @@ -165,7 +190,9 @@ function startLockHeartbeat (cacheKey) { function isLockStale (cacheKey) { try { const content = fs.readFileSync(getLockPath(cacheKey), 'utf8') - return Date.now() - Number(content) > CACHE_LOCK_TIMEOUT_MS + const timestamp = Number(content) + const now = Date.now() + return !isValidTimestamp(timestamp, now) || now - timestamp > CACHE_LOCK_TIMEOUT_MS } catch { return true } @@ -175,30 +202,37 @@ function isLockStale (cacheKey) { * Polls until the cache file appears or the timeout is reached. * * @param {string} cacheKey - * @param {Function} fetchFn - function(done) that fetches from the API + * @param {Function} fetchFn - function(cacheKey, done) that fetches from the API * @param {Function} done - callback(err, ...results) + * @param {number} cacheTtlMs - Maximum cache age in milliseconds. */ -function waitForCache (cacheKey, fetchFn, done) { +function waitForCache (cacheKey, fetchFn, done, cacheTtlMs) { const poll = () => { - const cached = readFromCache(cacheKey) + const cached = readFromCache(cacheKey, cacheTtlMs) if (cached) { return done(null, cached.data) } if (isLockStale(cacheKey)) { log.debug('%s lock is stale, attempting takeover', cacheKey) - releaseLock(cacheKey) - if (!tryAcquireLock(cacheKey)) { + if (!releaseLock(cacheKey)) { + return fetchFn(null, done) + } + const isLockOwner = tryAcquireLock(cacheKey) + if (isLockOwner === undefined) { + return fetchFn(null, done) + } + if (!isLockOwner) { return setTimeout(poll, CACHE_LOCK_POLL_MS) } - const cachedAfterTakeover = readFromCache(cacheKey) + const cachedAfterTakeover = readFromCache(cacheKey, cacheTtlMs) if (cachedAfterTakeover) { releaseLock(cacheKey) return done(null, cachedAfterTakeover.data) } const stopHeartbeat = startLockHeartbeat(cacheKey) - return fetchFn((err, ...results) => { + return fetchFn(cacheKey, (err, ...results) => { stopHeartbeat() done(err, ...results) }) @@ -218,14 +252,15 @@ function waitForCache (cacheKey, fetchFn, done) { * @param {Function} fetchFn - function(cacheKey, done) that performs the API request. * Must call writeToCache(cacheKey, data) on success before calling done(null, data). * @param {Function} done - callback(err, ...results) + * @param {number} [cacheTtlMs] - Maximum cache age in milliseconds. */ -function withCache (cacheKey, fetchFn, done) { +function withCache (cacheKey, fetchFn, done, cacheTtlMs = CACHE_TTL_MS) { if (!isCacheEnabled()) { return fetchFn(null, done) } // Fast path: cache hit - const cached = readFromCache(cacheKey) + const cached = readFromCache(cacheKey, cacheTtlMs) if (cached) { return done(null, cached.data) } @@ -233,9 +268,13 @@ function withCache (cacheKey, fetchFn, done) { // Try to become the fetcher (lock owner) const isLockOwner = tryAcquireLock(cacheKey) + if (isLockOwner === undefined) { + return fetchFn(null, done) + } + if (!isLockOwner) { log.debug('%s lock held by another process, waiting for cache', cacheKey) - return waitForCache(cacheKey, (cb) => fetchFn(cacheKey, cb), done) + return waitForCache(cacheKey, fetchFn, done, cacheTtlMs) } // This process owns the lock — start heartbeat and fetch diff --git a/packages/dd-trace/src/ci-visibility/requests/get-library-configuration.js b/packages/dd-trace/src/ci-visibility/requests/get-library-configuration.js index c21737e8865..b7d7a0a6586 100644 --- a/packages/dd-trace/src/ci-visibility/requests/get-library-configuration.js +++ b/packages/dd-trace/src/ci-visibility/requests/get-library-configuration.js @@ -14,7 +14,6 @@ const { TELEMETRY_GIT_REQUESTS_SETTINGS_ERRORS, TELEMETRY_GIT_REQUESTS_SETTINGS_RESPONSE, } = require('../telemetry') -const { writeSettingsToCache } = require('../test-optimization-cache') const { MAX_RETRIES, validateSettingsResponse } = require('../test-optimization-http-cache-schema') const request = require('./request') @@ -286,8 +285,6 @@ function getLibraryConfiguration ({ incrementCountMetric(TELEMETRY_GIT_REQUESTS_SETTINGS_RESPONSE, settings) - writeSettingsToCache(settings) - done(null, settings) } catch (err) { done(err) diff --git a/packages/dd-trace/test/ci-visibility/exporters/ci-visibility-exporter.spec.js b/packages/dd-trace/test/ci-visibility/exporters/ci-visibility-exporter.spec.js index 3a1efc30473..0aa8229582d 100644 --- a/packages/dd-trace/test/ci-visibility/exporters/ci-visibility-exporter.spec.js +++ b/packages/dd-trace/test/ci-visibility/exporters/ci-visibility-exporter.spec.js @@ -198,6 +198,24 @@ describe('CI Visibility Exporter', () => { assert.strictEqual(scope.isDone(), false) }) + it('should start the git upload timeout when an upload is requested', async () => { + const clock = sinon.useFakeTimers() + const ciVisibilityExporter = new CiVisibilityExporter({ + url, + testOptimization: { DD_CIVISIBILITY_GIT_UPLOAD_ENABLED: true }, + }) + + try { + ciVisibilityExporter.sendGitMetadata() + await clock.tickAsync(60_000) + + const err = await ciVisibilityExporter._gitUploadPromise + assert.match(err.message, /Timeout while uploading git metadata/) + } finally { + clock.restore() + } + }) + it('should resolve _gitUploadPromise when git metadata is fetched', (done) => { const scope = nock(url) .post('/api/v2/git/repository/search_commits') @@ -416,6 +434,52 @@ describe('CI Visibility Exporter', () => { }) ciVisibilityExporter._resolveCanUseCiVisProtocol(true) }) + it('does not request skippable suites when git metadata upload fails with require_git false', (done) => { + // Regression: the live settings path starts `sendGitMetadata` even when the + // backend returns `require_git: false`. `getSkippableSuites` awaits + // `_gitUploadPromise`, so a failed upload must resolve that promise with the + // error and suppress the skippable request. The settings fs-cache apply path + // must NOT call `_resolveGit()` on the live path, or it races the in-flight + // upload and masks its error. + nock(url) + .post('/api/v2/libraries/tests/services/setting') + .reply(200, JSON.stringify({ + data: { + attributes: { + itr_enabled: true, + require_git: false, + code_coverage: true, + tests_skipping: true, + }, + }, + })) + + const skippableScope = nock(url) + .post('/api/v2/ci/tests/skippable') + .reply(200, JSON.stringify({ data: [] })) + + const ciVisibilityExporter = new CiVisibilityExporter({ + url, + testOptimization: { DD_CIVISIBILITY_ITR_ENABLED: true }, + }) + // Simulate a failed git metadata upload: resolve the git promise with an error, + // as the real `sendGitMetadata` would on a non-2xx response. + ciVisibilityExporter.sendGitMetadata = function () { + setImmediate(() => this._resolveGit(new Error('git metadata upload failed'))) + } + ciVisibilityExporter._resolveCanUseCiVisProtocol(true) + + ciVisibilityExporter.getLibraryConfiguration({}, (settingsErr) => { + assert.strictEqual(settingsErr, null) + assert.strictEqual(ciVisibilityExporter.shouldRequestSkippableSuites(), true) + ciVisibilityExporter.getSkippableSuites({}, (skippableErr, skippableSuites) => { + assert.ok(skippableErr instanceof Error, 'skippable should surface the git upload error') + assert.deepStrictEqual(skippableSuites, []) + assert.strictEqual(skippableScope.isDone(), false, 'should NOT request skippable when git upload fails') + done() + }) + }) + }) it('will retry ITR configuration request if require_git is true', (done) => { const TIME_TO_UPLOAD_GIT = 50 let hasUploadedGit = false @@ -500,6 +564,49 @@ describe('CI Visibility Exporter', () => { }) ciVisibilityExporter._resolveGit() }) + it('clears phase-one settings when the post-upload settings request fails', (done) => { + // Regression: when the backend returns require_git:true and the second (post-upload) + // settings request fails, _libraryConfig must be reset to empty settings so stale + // phase-one feature flags don't stay installed (shouldRequestSkippableSuites etc.). + const scope = nock(url) + .post('/api/v2/libraries/tests/services/setting') + .reply(200, JSON.stringify({ + data: { + attributes: { + require_git: true, + code_coverage: true, + tests_skipping: true, + itr_enabled: true, + }, + }, + })) + .post('/api/v2/libraries/tests/services/setting') + .reply(400, JSON.stringify({ errors: [{ detail: 'backend error' }] })) + + const ciVisibilityExporter = new CiVisibilityExporter({ + url, + testOptimization: { + DD_CIVISIBILITY_ITR_ENABLED: true, + DD_CIVISIBILITY_GIT_UPLOAD_ENABLED: true, + }, + }) + sinon.stub(ciVisibilityExporter, 'sendGitMetadata') + ciVisibilityExporter._resolveCanUseCiVisProtocol(true) + ciVisibilityExporter.getLibraryConfiguration({}, (err, libraryConfig) => { + assert.strictEqual(scope.isDone(), true, 'both phases should have hit the API') + assert.ok(err, 'should surface the second-request error') + // Phase-1 had tests_skipping:true and itr_enabled:true; after the failure these + // must NOT remain installed on _libraryConfig. + assert.strictEqual( + ciVisibilityExporter.shouldRequestSkippableSuites(), + false, + 'stale phase-1 flags must not enable skippable after a failed negotiation' + ) + done() + }) + // Simulate the git upload finishing so the phase-2 request can proceed. + setImmediate(() => ciVisibilityExporter._resolveGit()) + }) }) }) diff --git a/packages/dd-trace/test/ci-visibility/exporters/ci-visibility-settings-fs-cache.spec.js b/packages/dd-trace/test/ci-visibility/exporters/ci-visibility-settings-fs-cache.spec.js new file mode 100644 index 00000000000..824ec8630f3 --- /dev/null +++ b/packages/dd-trace/test/ci-visibility/exporters/ci-visibility-settings-fs-cache.spec.js @@ -0,0 +1,885 @@ +'use strict' + +const assert = require('node:assert/strict') +const fs = require('node:fs') +const os = require('node:os') +const path = require('node:path') + +const { describe, it, beforeEach, afterEach } = require('mocha') +const nock = require('nock') +const sinon = require('sinon') + +require('../../../../dd-trace/test/setup/core') +const getConfig = require('../../../src/config') +const { defaults: { hostname, port } } = require('../../../src/config/defaults') +const { parseLibraryConfigurationResponse } = + require('../../../src/ci-visibility/requests/get-library-configuration') +const { buildCacheKey, getCachePath, getLockPath, withCache } = + require('../../../src/ci-visibility/requests/fs-cache') +const CiVisibilityExporter = + require('../../../src/ci-visibility/exporters/ci-visibility-exporter') +const buildSettingsCacheKey = require('../../../src/ci-visibility/exporters/settings-cache-key') + +const url = new URL(`http://${hostname}:${port}`) +const SETTINGS_CACHE_TTL_MS = 5 * 60 * 1000 + +const TEST_CONFIGURATION = { + repositoryUrl: 'git@github.com:Datadog/dd-trace-js.git', + sha: 'deadbeefdeadbeefdeadbeefdeadbeefdeadbeef', + branch: 'main', + testLevel: 'suite', + osPlatform: 'darwin', + osVersion: '22.0', + osArchitecture: 'arm64', + runtimeName: 'nodejs', + runtimeVersion: '18.0.0', +} + +const SETTINGS_NO_GIT = { + data: { + attributes: { + itr_enabled: true, + require_git: false, + code_coverage: true, + tests_skipping: true, + known_tests_enabled: false, + }, + }, +} + +const SETTINGS_REQUIRE_GIT = { + data: { + attributes: { + itr_enabled: true, + require_git: true, + code_coverage: true, + tests_skipping: true, + known_tests_enabled: false, + // Phase 1 disables coverage report upload; phase 2 enables it. nyc reads + // this field off the settings handoff file, so the final file must hold + // the phase-2 value, never the transient phase-1 value. + coverage_report_upload_enabled: false, + }, + }, +} + +const SETTINGS_FINAL_AFTER_GIT = { + data: { + attributes: { + itr_enabled: true, + require_git: false, + code_coverage: true, + tests_skipping: true, + known_tests_enabled: false, + coverage_report_upload_enabled: true, + }, + }, +} + +const SETTINGS_NO_SKIPPING = { + data: { + attributes: { + itr_enabled: true, + require_git: false, + code_coverage: false, + tests_skipping: false, + known_tests_enabled: false, + }, + }, +} + +function cacheKeyForConfiguration (exporter, testConfiguration) { + const configuration = exporter.getRequestConfiguration(testConfiguration) + return buildSettingsCacheKey(configuration) +} + +function cleanup (exporter, testConfiguration) { + const key = cacheKeyForConfiguration(exporter, testConfiguration) + try { fs.unlinkSync(getCachePath(key)) } catch { /* ignore */ } + try { fs.unlinkSync(getLockPath(key)) } catch { /* ignore */ } +} + +function makeExporter (testOptimization, exporterUrl = url) { + return new CiVisibilityExporter({ + url: exporterUrl, + env: 'test', + service: 'dd-trace-js', + testOptimization, + }) +} + +function requestLibraryConfiguration (exporter) { + return new Promise((resolve, reject) => { + exporter.getLibraryConfiguration(TEST_CONFIGURATION, (err, libraryConfig) => { + if (err) return reject(err) + resolve(libraryConfig) + }) + }) +} + +describe('ci-visibility settings filesystem cache', () => { + let originalApiKey + let originalEnvApiKey + let originalEnvFsCache + let originalFsCache + let originalSettingsCachePath + let originalForceCoverage + let originalForceTestSkipping + let originalCoverageReportUpload + let settingsCacheDir + let settingsCachePath + + beforeEach(() => { + originalApiKey = getConfig().DD_API_KEY + originalEnvApiKey = process.env.DD_API_KEY + originalEnvFsCache = process.env.DD_EXPERIMENTAL_TEST_REQUESTS_FS_CACHE + originalFsCache = getConfig().DD_EXPERIMENTAL_TEST_REQUESTS_FS_CACHE + originalSettingsCachePath = process.env.DD_EXPERIMENTAL_TEST_OPT_SETTINGS_CACHE + originalForceCoverage = getConfig().testOptimization.DD_CIVISIBILITY_DANGEROUSLY_FORCE_COVERAGE + originalForceTestSkipping = getConfig().testOptimization.DD_CIVISIBILITY_DANGEROUSLY_FORCE_TEST_SKIPPING + originalCoverageReportUpload = + getConfig().testOptimization.DD_CIVISIBILITY_CODE_COVERAGE_REPORT_UPLOAD_ENABLED + getConfig().DD_API_KEY = '1' + getConfig().testOptimization.DD_CIVISIBILITY_DANGEROUSLY_FORCE_COVERAGE = false + getConfig().testOptimization.DD_CIVISIBILITY_DANGEROUSLY_FORCE_TEST_SKIPPING = false + getConfig().testOptimization.DD_CIVISIBILITY_CODE_COVERAGE_REPORT_UPLOAD_ENABLED = true + + process.env.DD_API_KEY = '1' + + process.env.DD_EXPERIMENTAL_TEST_REQUESTS_FS_CACHE = 'true' + getConfig().DD_EXPERIMENTAL_TEST_REQUESTS_FS_CACHE = true + settingsCacheDir = fs.mkdtempSync(path.join(os.tmpdir(), 'dd-js-settings-handoff-')) + settingsCachePath = path.join(settingsCacheDir, 'nyc-settings.json') + process.env.DD_EXPERIMENTAL_TEST_OPT_SETTINGS_CACHE = settingsCachePath + nock.cleanAll() + }) + + afterEach(() => { + getConfig().DD_API_KEY = originalApiKey + getConfig().DD_EXPERIMENTAL_TEST_REQUESTS_FS_CACHE = originalFsCache + getConfig().testOptimization.DD_CIVISIBILITY_DANGEROUSLY_FORCE_COVERAGE = originalForceCoverage + getConfig().testOptimization.DD_CIVISIBILITY_DANGEROUSLY_FORCE_TEST_SKIPPING = originalForceTestSkipping + getConfig().testOptimization.DD_CIVISIBILITY_CODE_COVERAGE_REPORT_UPLOAD_ENABLED = + originalCoverageReportUpload + if (originalSettingsCachePath === undefined) { + delete process.env.DD_EXPERIMENTAL_TEST_OPT_SETTINGS_CACHE + } else { + process.env.DD_EXPERIMENTAL_TEST_OPT_SETTINGS_CACHE = originalSettingsCachePath + } + + if (originalEnvApiKey === undefined) { + delete process.env.DD_API_KEY + } else { + process.env.DD_API_KEY = originalEnvApiKey + } + if (originalEnvFsCache === undefined) { + delete process.env.DD_EXPERIMENTAL_TEST_REQUESTS_FS_CACHE + } else { + process.env.DD_EXPERIMENTAL_TEST_REQUESTS_FS_CACHE = originalEnvFsCache + } + if (settingsCacheDir) { + fs.rmSync(settingsCacheDir, { recursive: true, force: true }) + } + nock.cleanAll() + }) + + it('serves a second call from the filesystem cache without hitting the API', (done) => { + const exporter = makeExporter({ DD_CIVISIBILITY_ITR_ENABLED: true }) + exporter._resolveCanUseCiVisProtocol(true) + cleanup(exporter, TEST_CONFIGURATION) + + const firstScope = nock(url) + .post('/api/v2/libraries/tests/services/setting') + .reply(200, JSON.stringify(SETTINGS_NO_GIT)) + + exporter.getLibraryConfiguration(TEST_CONFIGURATION, (err, firstConfig) => { + assert.strictEqual(err, null) + assert.strictEqual(firstScope.isDone(), true) + assert.strictEqual(firstConfig.requireGit, false) + + const secondScope = nock(url) + .post('/api/v2/libraries/tests/services/setting') + .reply(200, JSON.stringify(SETTINGS_NO_GIT)) + + exporter.getLibraryConfiguration(TEST_CONFIGURATION, (err, secondConfig) => { + assert.strictEqual(err, null) + assert.strictEqual(secondScope.isDone(), false, 'API should NOT be called on a cache hit') + assert.strictEqual(secondConfig.requireGit, false) + cleanup(exporter, TEST_CONFIGURATION) + done() + }) + }) + }) + + it('writes the final configuration to the cache file on a cache miss', (done) => { + const exporter = makeExporter({ DD_CIVISIBILITY_ITR_ENABLED: true }) + exporter._resolveCanUseCiVisProtocol(true) + cleanup(exporter, TEST_CONFIGURATION) + + nock(url) + .post('/api/v2/libraries/tests/services/setting') + .reply(200, JSON.stringify(SETTINGS_NO_GIT)) + + exporter.getLibraryConfiguration(TEST_CONFIGURATION, (err) => { + assert.strictEqual(err, null) + + const key = cacheKeyForConfiguration(exporter, TEST_CONFIGURATION) + assert.ok(fs.existsSync(getCachePath(key)), 'cache file should exist') + assert.strictEqual(fs.existsSync(getLockPath(key)), false, 'lock should be cleaned up') + + const cached = JSON.parse(fs.readFileSync(getCachePath(key), 'utf8')) + assert.strictEqual(cached.data.requireGit, false) + cleanup(exporter, TEST_CONFIGURATION) + done() + }) + }) + + for (const { description, age, shouldFetch } of [ + { + description: 'accepts settings at the five-minute cache TTL boundary', + age: SETTINGS_CACHE_TTL_MS, + shouldFetch: false, + }, + { + description: 'refreshes settings immediately after the five-minute cache TTL boundary', + age: SETTINGS_CACHE_TTL_MS + 1, + shouldFetch: true, + }, + ]) { + it(description, async () => { + const clock = sinon.useFakeTimers({ + now: SETTINGS_CACHE_TTL_MS * 2, + toFake: ['Date'], + }) + const exporter = makeExporter({ DD_CIVISIBILITY_ITR_ENABLED: true }) + exporter._resolveCanUseCiVisProtocol(true) + cleanup(exporter, TEST_CONFIGURATION) + + const key = cacheKeyForConfiguration(exporter, TEST_CONFIGURATION) + const cachedSettings = parseLibraryConfigurationResponse(SETTINGS_NO_SKIPPING) + fs.writeFileSync( + getCachePath(key), + JSON.stringify({ timestamp: Date.now() - age, data: cachedSettings }), + 'utf8' + ) + const scope = nock(url) + .post('/api/v2/libraries/tests/services/setting') + .reply(200, JSON.stringify(SETTINGS_NO_GIT)) + + try { + const libraryConfig = await requestLibraryConfiguration(exporter) + assert.strictEqual(scope.isDone(), shouldFetch) + assert.strictEqual(libraryConfig.isSuitesSkippingEnabled, shouldFetch) + } finally { + clock.restore() + cleanup(exporter, TEST_CONFIGURATION) + } + }) + } + + it('isolates filesystem settings by backend origin', async () => { + const otherUrl = new URL(`http://localhost:${port}`) + const firstExporter = makeExporter({ DD_CIVISIBILITY_ITR_ENABLED: true }) + const secondExporter = makeExporter({ DD_CIVISIBILITY_ITR_ENABLED: true }, otherUrl) + firstExporter._resolveCanUseCiVisProtocol(true) + secondExporter._resolveCanUseCiVisProtocol(true) + cleanup(firstExporter, TEST_CONFIGURATION) + cleanup(secondExporter, TEST_CONFIGURATION) + + try { + nock(url) + .post('/api/v2/libraries/tests/services/setting') + .reply(200, JSON.stringify(SETTINGS_NO_GIT)) + await requestLibraryConfiguration(firstExporter) + + const secondScope = nock(otherUrl) + .post('/api/v2/libraries/tests/services/setting') + .reply(200, JSON.stringify(SETTINGS_NO_SKIPPING)) + + const secondConfig = await requestLibraryConfiguration(secondExporter) + assert.strictEqual(secondScope.isDone(), true, 'the second backend should fetch its own settings') + assert.strictEqual(secondConfig.isSuitesSkippingEnabled, false) + } finally { + cleanup(firstExporter, TEST_CONFIGURATION) + cleanup(secondExporter, TEST_CONFIGURATION) + } + }) + + it('isolates Unix agent endpoints by socket path', () => { + const firstExporter = makeExporter( + { DD_CIVISIBILITY_ITR_ENABLED: true }, + new URL('unix:///tmp/dd-agent-one.sock') + ) + const secondExporter = makeExporter( + { DD_CIVISIBILITY_ITR_ENABLED: true }, + new URL('unix:///tmp/dd-agent-two.sock') + ) + + const firstConfiguration = firstExporter.getRequestConfiguration(TEST_CONFIGURATION) + const secondConfiguration = secondExporter.getRequestConfiguration(TEST_CONFIGURATION) + assert.strictEqual(firstConfiguration.url.origin, 'null') + assert.strictEqual(secondConfiguration.url.origin, 'null') + assert.notStrictEqual( + cacheKeyForConfiguration(firstExporter, TEST_CONFIGURATION), + cacheKeyForConfiguration(secondExporter, TEST_CONFIGURATION) + ) + + firstExporter._resolveCanUseCiVisProtocol(false) + secondExporter._resolveCanUseCiVisProtocol(false) + }) + + it('isolates parsed settings by local override flags', async () => { + const firstExporter = makeExporter({ DD_CIVISIBILITY_ITR_ENABLED: true }) + const secondExporter = makeExporter({ DD_CIVISIBILITY_ITR_ENABLED: true }) + firstExporter._resolveCanUseCiVisProtocol(true) + secondExporter._resolveCanUseCiVisProtocol(true) + + try { + getConfig().testOptimization.DD_CIVISIBILITY_DANGEROUSLY_FORCE_COVERAGE = true + cleanup(firstExporter, TEST_CONFIGURATION) + nock(url) + .post('/api/v2/libraries/tests/services/setting') + .reply(200, JSON.stringify(SETTINGS_NO_SKIPPING)) + + const firstConfig = await requestLibraryConfiguration(firstExporter) + assert.strictEqual(firstConfig.isCodeCoverageEnabled, true) + + getConfig().testOptimization.DD_CIVISIBILITY_DANGEROUSLY_FORCE_COVERAGE = false + cleanup(secondExporter, TEST_CONFIGURATION) + const secondScope = nock(url) + .post('/api/v2/libraries/tests/services/setting') + .reply(200, JSON.stringify(SETTINGS_NO_SKIPPING)) + + const secondConfig = await requestLibraryConfiguration(secondExporter) + assert.strictEqual(secondScope.isDone(), true, 'the second local configuration should fetch its own settings') + assert.strictEqual(secondConfig.isCodeCoverageEnabled, false) + } finally { + getConfig().testOptimization.DD_CIVISIBILITY_DANGEROUSLY_FORCE_COVERAGE = false + cleanup(secondExporter, TEST_CONFIGURATION) + getConfig().testOptimization.DD_CIVISIBILITY_DANGEROUSLY_FORCE_COVERAGE = true + cleanup(firstExporter, TEST_CONFIGURATION) + getConfig().testOptimization.DD_CIVISIBILITY_DANGEROUSLY_FORCE_COVERAGE = false + } + }) + + it('only caches the final (post-git-upload) configuration when require_git is true', (done) => { + const exporter = makeExporter({ + DD_CIVISIBILITY_ITR_ENABLED: true, + DD_CIVISIBILITY_GIT_UPLOAD_ENABLED: true, + }) + // Avoid a real git upload; we resolve the upload promise ourselves below. + exporter.sendGitMetadata = function () {} + exporter._resolveCanUseCiVisProtocol(true) + cleanup(exporter, TEST_CONFIGURATION) + + const scope = nock(url) + .post('/api/v2/libraries/tests/services/setting') + .reply(200, JSON.stringify(SETTINGS_REQUIRE_GIT)) + .post('/api/v2/libraries/tests/services/setting') + .reply(200, JSON.stringify(SETTINGS_NO_GIT)) + + exporter.getLibraryConfiguration(TEST_CONFIGURATION, (err, firstConfig) => { + assert.strictEqual(err, null) + assert.strictEqual(scope.isDone(), true, 'both phases should have hit the API') + assert.strictEqual(firstConfig.requireGit, false, 'final config should have require_git false') + + const key = cacheKeyForConfiguration(exporter, TEST_CONFIGURATION) + const cached = JSON.parse(fs.readFileSync(getCachePath(key), 'utf8')) + assert.strictEqual(cached.data.requireGit, false, 'cache must hold the final config only') + + // A second call must be served from cache and must not see the phase-1 require_git:true. + const secondScope = nock(url) + .post('/api/v2/libraries/tests/services/setting') + .reply(200, JSON.stringify(SETTINGS_NO_GIT)) + + exporter.getLibraryConfiguration(TEST_CONFIGURATION, (err, secondConfig) => { + assert.strictEqual(err, null) + assert.strictEqual(secondScope.isDone(), false, 'API should NOT be called on a cache hit') + assert.strictEqual(secondConfig.requireGit, false) + cleanup(exporter, TEST_CONFIGURATION) + done() + }) + }) + // Simulate the git upload finishing so the phase-2 request can proceed. + setImmediate(() => exporter._resolveGit()) + }) + + // Regression: nyc runs in a separate process and reads the library + // configuration from the file pointed at by DD_EXPERIMENTAL_TEST_OPT_SETTINGS_CACHE + // (see datadog-plugin-nyc readLibraryConfiguration). The settings request has a + // two-phase shape (require_git -> upload git metadata -> re-request), and the + // handoff file must end up holding the final (phase-2) configuration, never the + // transient phase-1 state, so nyc sees the correct coverage-report-upload flag. + it('writes only the final config to the nyc settings handoff file across the require_git flow', (done) => { + const exporter = makeExporter({ + DD_CIVISIBILITY_ITR_ENABLED: true, + DD_CIVISIBILITY_GIT_UPLOAD_ENABLED: true, + }) + // Avoid a real git upload; we resolve the upload promise ourselves below. + exporter.sendGitMetadata = function () {} + exporter._resolveCanUseCiVisProtocol(true) + cleanup(exporter, TEST_CONFIGURATION) + + const scope = nock(url) + .post('/api/v2/libraries/tests/services/setting') + .reply(200, JSON.stringify(SETTINGS_REQUIRE_GIT)) + .post('/api/v2/libraries/tests/services/setting') + .reply(200, JSON.stringify(SETTINGS_FINAL_AFTER_GIT)) + + exporter.getLibraryConfiguration(TEST_CONFIGURATION, (err, finalConfig) => { + try { + assert.strictEqual(err, null) + assert.strictEqual(scope.isDone(), true, 'both phases should have hit the API') + assert.strictEqual(finalConfig.requireGit, false, 'final config should have require_git false') + assert.strictEqual( + finalConfig.isCoverageReportUploadEnabled, + true, + 'final config should enable coverage report upload' + ) + + assert.ok(fs.existsSync(settingsCachePath), 'nyc settings handoff file should exist') + const handoff = JSON.parse(fs.readFileSync(settingsCachePath, 'utf8')) + // The handoff file must reflect the final (phase-2) configuration, not the + // transient phase-1 state where coverage report upload was disabled. + assert.strictEqual(handoff.requireGit, false, 'handoff must not retain phase-1 require_git') + assert.strictEqual( + handoff.isCoverageReportUploadEnabled, + true, + 'handoff must hold the phase-2 coverage report upload flag for nyc' + ) + cleanup(exporter, TEST_CONFIGURATION) + done() + } catch (err) { + cleanup(exporter, TEST_CONFIGURATION) + done(err) + } + }) + // Simulate the git upload finishing so the phase-2 request can proceed. + setImmediate(() => exporter._resolveGit()) + }) + + it('preserves the git upload gate for settings filesystem cache consumers', async () => { + const exporter = makeExporter({ + DD_CIVISIBILITY_ITR_ENABLED: true, + DD_CIVISIBILITY_GIT_UPLOAD_ENABLED: true, + }) + exporter._resolveCanUseCiVisProtocol(true) + cleanup(exporter, TEST_CONFIGURATION) + + const key = cacheKeyForConfiguration(exporter, TEST_CONFIGURATION) + const cachedSettings = parseLibraryConfigurationResponse(SETTINGS_NO_GIT) + fs.writeFileSync( + getCachePath(key), + JSON.stringify({ timestamp: Date.now(), data: cachedSettings }), + 'utf8' + ) + + let gitUploadStarted = false + exporter.sendGitMetadata = function () { + gitUploadStarted = true + } + + try { + const gitUploadResult = exporter._gitUploadPromise.then(error => ({ settled: true, error })) + const libraryConfig = await requestLibraryConfiguration(exporter) + const pendingResult = await Promise.race([ + gitUploadResult, + Promise.resolve({ settled: false }), + ]) + + assert.strictEqual(libraryConfig.requireGit, false) + assert.strictEqual(gitUploadStarted, true, 'a cache consumer must start its own git upload') + assert.strictEqual(pendingResult.settled, false, 'the cache hit must not release the git gate') + + exporter._resolveGit() + const { error } = await gitUploadResult + assert.strictEqual(error, undefined) + } finally { + cleanup(exporter, TEST_CONFIGURATION) + } + }) + + it('does not consume the git upload timeout while waiting for the settings cache', async () => { + const clock = sinon.useFakeTimers() + const exporter = makeExporter({ + DD_CIVISIBILITY_ITR_ENABLED: true, + DD_CIVISIBILITY_GIT_UPLOAD_ENABLED: false, + }) + exporter._resolveCanUseCiVisProtocol(true) + cleanup(exporter, TEST_CONFIGURATION) + + const key = cacheKeyForConfiguration(exporter, TEST_CONFIGURATION) + fs.writeFileSync(getLockPath(key), String(Date.now()), 'utf8') + + try { + const settingsRequest = requestLibraryConfiguration(exporter) + await clock.tickAsync(60_000) + + const cachedSettings = parseLibraryConfigurationResponse(SETTINGS_NO_GIT) + fs.writeFileSync( + getCachePath(key), + JSON.stringify({ timestamp: Date.now(), data: cachedSettings }), + 'utf8' + ) + await clock.tickAsync(500) + + const libraryConfig = await settingsRequest + const gitUploadError = await exporter._gitUploadPromise + assert.strictEqual(libraryConfig.requireGit, false) + assert.strictEqual(gitUploadError, undefined) + } finally { + clock.restore() + cleanup(exporter, TEST_CONFIGURATION) + } + }) + + const validCachedSettings = parseLibraryConfigurationResponse(SETTINGS_NO_GIT) + for (const [description, data] of [ + ['null', null], + ['an array', []], + ['an incomplete object', {}], + ['failed test replay without flaky test retries', { + ...validCachedSettings, + isDiEnabled: true, + isFlakyTestRetriesEnabled: false, + }], + ['early flake detection without known tests', { + ...validCachedSettings, + isEarlyFlakeDetectionEnabled: true, + isKnownTestsEnabled: false, + }], + ['an incomplete EFD retry policy', { + ...validCachedSettings, + isEarlyFlakeDetectionEnabled: true, + earlyFlakeDetectionRetryPolicy: { schedulingRetryCount: 1 }, + }], + ['a null EFD retry policy', { + ...validCachedSettings, + earlyFlakeDetectionRetryPolicy: null, + }], + ['an invalid EFD scheduling retry count', { + ...validCachedSettings, + earlyFlakeDetectionRetryPolicy: { + ...validCachedSettings.earlyFlakeDetectionRetryPolicy, + schedulingRetryCount: -1, + }, + }], + ['an inconsistent EFD scheduling retry count', { + ...validCachedSettings, + earlyFlakeDetectionRetryPolicy: { + ...validCachedSettings.earlyFlakeDetectionRetryPolicy, + schedulingRetryCount: 1, + }, + }], + ['an invalid EFD duration retry entry', { + ...validCachedSettings, + earlyFlakeDetectionRetryPolicy: { + ...validCachedSettings.earlyFlakeDetectionRetryPolicy, + durationRetryCounts: [ + null, + ...validCachedSettings.earlyFlakeDetectionRetryPolicy.durationRetryCounts.slice(1), + ], + }, + }], + ['an excessive EFD duration retry count', { + ...validCachedSettings, + earlyFlakeDetectionRetryPolicy: { + schedulingRetryCount: 101, + durationRetryCounts: [ + { + ...validCachedSettings.earlyFlakeDetectionRetryPolicy.durationRetryCounts[0], + retryCount: 101, + }, + ...validCachedSettings.earlyFlakeDetectionRetryPolicy.durationRetryCounts.slice(1), + ], + }, + }], + ['an invalid EFD faulty threshold', { + ...validCachedSettings, + earlyFlakeDetectionFaultyThreshold: 101, + }], + ['an excessive test management retry count', { + ...validCachedSettings, + testManagementAttemptToFixRetries: 101, + }], + ]) { + it(`treats malformed cache data (${description}) as a miss and falls back to the API`, (done) => { + const exporter = makeExporter({ DD_CIVISIBILITY_ITR_ENABLED: true }) + exporter._resolveCanUseCiVisProtocol(true) + cleanup(exporter, TEST_CONFIGURATION) + + const key = cacheKeyForConfiguration(exporter, TEST_CONFIGURATION) + fs.writeFileSync( + getCachePath(key), + JSON.stringify({ timestamp: Date.now(), data }), + 'utf8' + ) + + const scope = nock(url) + .post('/api/v2/libraries/tests/services/setting') + .reply(200, JSON.stringify(SETTINGS_NO_GIT)) + + exporter.getLibraryConfiguration(TEST_CONFIGURATION, (err, libraryConfig) => { + try { + assert.strictEqual(err, null, 'malformed cache must not surface an error') + assert.ok(libraryConfig, 'malformed cache must fall back to a real config') + assert.strictEqual(libraryConfig.requireGit, false) + assert.strictEqual(scope.isDone(), true, 'malformed cache should fall back to the API') + cleanup(exporter, TEST_CONFIGURATION) + done() + } catch (err) { + cleanup(exporter, TEST_CONFIGURATION) + done(err) + } + }) + }) + } + + for (const [description, getTimestamp] of [ + ['a missing timestamp', () => undefined], + ['a nonnumeric timestamp', () => 'not-a-number'], + ['a future timestamp', () => Date.now() + 60_000], + ]) { + it(`treats cache data with ${description} as a miss and falls back to the API`, (done) => { + const exporter = makeExporter({ DD_CIVISIBILITY_ITR_ENABLED: true }) + exporter._resolveCanUseCiVisProtocol(true) + cleanup(exporter, TEST_CONFIGURATION) + + const key = cacheKeyForConfiguration(exporter, TEST_CONFIGURATION) + fs.writeFileSync( + getCachePath(key), + JSON.stringify({ timestamp: getTimestamp(), data: validCachedSettings }), + 'utf8' + ) + + const scope = nock(url) + .post('/api/v2/libraries/tests/services/setting') + .reply(200, JSON.stringify(SETTINGS_NO_GIT)) + + exporter.getLibraryConfiguration(TEST_CONFIGURATION, (err, libraryConfig) => { + try { + assert.strictEqual(err, null, 'invalid timestamp must not surface an error') + assert.strictEqual(libraryConfig.requireGit, false) + assert.strictEqual(scope.isDone(), true, 'invalid timestamp should fall back to the API') + cleanup(exporter, TEST_CONFIGURATION) + done() + } catch (err) { + cleanup(exporter, TEST_CONFIGURATION) + done(err) + } + }) + }) + } + + it('bypasses an invalid cache entry that cannot be deleted', async () => { + const exporter = makeExporter({ DD_CIVISIBILITY_ITR_ENABLED: true }) + exporter._resolveCanUseCiVisProtocol(true) + cleanup(exporter, TEST_CONFIGURATION) + + const key = cacheKeyForConfiguration(exporter, TEST_CONFIGURATION) + const cachePath = getCachePath(key) + fs.writeFileSync(cachePath, JSON.stringify({ timestamp: Date.now(), data: null }), 'utf8') + + const unlinkSync = fs.unlinkSync + const unlinkStub = sinon.stub(fs, 'unlinkSync').callsFake(path => { + if (path === cachePath) { + const err = new Error('permission denied') + err.code = 'EACCES' + throw err + } + return unlinkSync(path) + }) + const scope = nock(url) + .post('/api/v2/libraries/tests/services/setting') + .reply(200, JSON.stringify(SETTINGS_NO_GIT)) + + try { + const libraryConfig = await requestLibraryConfiguration(exporter) + assert.strictEqual(libraryConfig.requireGit, false) + assert.strictEqual(scope.isDone(), true, 'an undeletable invalid entry should be bypassed') + } finally { + unlinkStub.restore() + cleanup(exporter, TEST_CONFIGURATION) + } + }) + + it('bypasses the filesystem cache when its lock cannot be created', async () => { + const exporter = makeExporter({ DD_CIVISIBILITY_ITR_ENABLED: true }) + exporter._resolveCanUseCiVisProtocol(true) + cleanup(exporter, TEST_CONFIGURATION) + + const lockPath = getLockPath(cacheKeyForConfiguration(exporter, TEST_CONFIGURATION)) + const openSync = fs.openSync + const openStub = sinon.stub(fs, 'openSync').callsFake((filePath, ...args) => { + if (filePath === lockPath) { + const err = new Error('permission denied') + err.code = 'EACCES' + throw err + } + return openSync(filePath, ...args) + }) + const scope = nock(url) + .post('/api/v2/libraries/tests/services/setting') + .reply(200, JSON.stringify(SETTINGS_NO_GIT)) + + try { + const libraryConfig = await requestLibraryConfiguration(exporter) + assert.strictEqual(libraryConfig.requireGit, false) + assert.strictEqual(scope.isDone(), true, 'an unavailable lock should bypass the cache') + } finally { + openStub.restore() + cleanup(exporter, TEST_CONFIGURATION) + } + }) + + it('bypasses the filesystem cache when a stale lock cannot be deleted', async () => { + const exporter = makeExporter({ DD_CIVISIBILITY_ITR_ENABLED: true }) + exporter._resolveCanUseCiVisProtocol(true) + cleanup(exporter, TEST_CONFIGURATION) + + const lockPath = getLockPath(cacheKeyForConfiguration(exporter, TEST_CONFIGURATION)) + fs.writeFileSync(lockPath, '0', 'utf8') + const unlinkSync = fs.unlinkSync + const unlinkStub = sinon.stub(fs, 'unlinkSync').callsFake(filePath => { + if (filePath === lockPath) { + const err = new Error('permission denied') + err.code = 'EACCES' + throw err + } + return unlinkSync(filePath) + }) + const scope = nock(url) + .post('/api/v2/libraries/tests/services/setting') + .reply(200, JSON.stringify(SETTINGS_NO_GIT)) + + try { + const libraryConfig = await requestLibraryConfiguration(exporter) + assert.strictEqual(libraryConfig.requireGit, false) + assert.strictEqual(scope.isDone(), true, 'an undeletable stale lock should bypass the cache') + } finally { + unlinkStub.restore() + cleanup(exporter, TEST_CONFIGURATION) + } + }) + + it('takes over a stale filesystem cache lock', async () => { + const exporter = makeExporter({ DD_CIVISIBILITY_ITR_ENABLED: true }) + exporter._resolveCanUseCiVisProtocol(true) + cleanup(exporter, TEST_CONFIGURATION) + + const key = cacheKeyForConfiguration(exporter, TEST_CONFIGURATION) + fs.writeFileSync(getLockPath(key), '0', 'utf8') + const scope = nock(url) + .post('/api/v2/libraries/tests/services/setting') + .reply(200, JSON.stringify(SETTINGS_NO_GIT)) + + try { + const libraryConfig = await requestLibraryConfiguration(exporter) + assert.strictEqual(libraryConfig.requireGit, false) + assert.strictEqual(scope.isDone(), true, 'the stale lock owner should fetch settings') + assert.strictEqual(fs.existsSync(getCachePath(key)), true, 'the stale lock owner should populate the cache') + assert.strictEqual(fs.existsSync(getLockPath(key)), false, 'the replacement lock should be released') + } finally { + cleanup(exporter, TEST_CONFIGURATION) + } + }) + + for (const [description, getTimestamp] of [ + ['a nonnumeric timestamp', () => 'not-a-number'], + ['a future timestamp', () => String(Date.now() + 60_000)], + ]) { + it(`takes over a filesystem cache lock with ${description}`, async () => { + const exporter = makeExporter({ DD_CIVISIBILITY_ITR_ENABLED: true }) + exporter._resolveCanUseCiVisProtocol(true) + cleanup(exporter, TEST_CONFIGURATION) + + const key = cacheKeyForConfiguration(exporter, TEST_CONFIGURATION) + fs.writeFileSync(getLockPath(key), getTimestamp(), 'utf8') + const scope = nock(url) + .post('/api/v2/libraries/tests/services/setting') + .reply(200, JSON.stringify(SETTINGS_NO_GIT)) + + try { + const libraryConfig = await requestLibraryConfiguration(exporter) + assert.strictEqual(libraryConfig.requireGit, false) + assert.strictEqual(scope.isDone(), true, 'the replacement lock owner should fetch settings') + assert.strictEqual(fs.existsSync(getCachePath(key)), true, 'the replacement lock owner should cache settings') + assert.strictEqual(fs.existsSync(getLockPath(key)), false, 'the replacement lock should be released') + } finally { + cleanup(exporter, TEST_CONFIGURATION) + } + }) + } + + it('bypasses the cache when a stale lock cannot be replaced', async () => { + const exporter = makeExporter({ DD_CIVISIBILITY_ITR_ENABLED: true }) + exporter._resolveCanUseCiVisProtocol(true) + cleanup(exporter, TEST_CONFIGURATION) + + const lockPath = getLockPath(cacheKeyForConfiguration(exporter, TEST_CONFIGURATION)) + fs.writeFileSync(lockPath, '0', 'utf8') + const openSync = fs.openSync + let lockOpenCount = 0 + const openStub = sinon.stub(fs, 'openSync').callsFake((filePath, ...args) => { + if (filePath === lockPath && ++lockOpenCount === 2) { + const err = new Error('permission denied') + err.code = 'EACCES' + throw err + } + return openSync(filePath, ...args) + }) + const scope = nock(url) + .post('/api/v2/libraries/tests/services/setting') + .reply(200, JSON.stringify(SETTINGS_NO_GIT)) + + try { + const libraryConfig = await requestLibraryConfiguration(exporter) + assert.strictEqual(libraryConfig.requireGit, false) + assert.strictEqual(scope.isDone(), true, 'a failed stale-lock takeover should bypass the cache') + } finally { + openStub.restore() + cleanup(exporter, TEST_CONFIGURATION) + } + }) + + it('retries when another process replaces a stale lock first', async () => { + const clock = sinon.useFakeTimers({ now: 1_000_000 }) + const key = buildCacheKey('settings-lock-race', [process.pid]) + const lockPath = getLockPath(key) + try { fs.unlinkSync(getCachePath(key)) } catch { /* ignore */ } + try { fs.unlinkSync(lockPath) } catch { /* ignore */ } + fs.writeFileSync(lockPath, '0', 'utf8') + const openSync = fs.openSync + let lockOpenCount = 0 + const openStub = sinon.stub(fs, 'openSync').callsFake((filePath, ...args) => { + if (filePath === lockPath && ++lockOpenCount === 2) { + const fd = openSync(filePath, fs.constants.O_CREAT | fs.constants.O_EXCL | fs.constants.O_WRONLY) + fs.writeSync(fd, '0') + fs.closeSync(fd) + const err = new Error('lock replaced by another process') + err.code = 'EEXIST' + throw err + } + return openSync(filePath, ...args) + }) + + try { + const cacheResult = new Promise((resolve, reject) => { + withCache(key, (activeCacheKey, done) => { + assert.strictEqual(activeCacheKey, key) + done(null, 'fetched') + }, (err, result) => { + if (err) return reject(err) + resolve(result) + }) + }) + await clock.tickAsync(500) + assert.strictEqual(await cacheResult, 'fetched') + } finally { + openStub.restore() + clock.restore() + try { fs.unlinkSync(getCachePath(key)) } catch { /* ignore */ } + try { fs.unlinkSync(lockPath) } catch { /* ignore */ } + } + }) +}) From d005afa2c7cd45ecac27f96768be7136411190bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Antonio=20Fern=C3=A1ndez=20de=20Alba?= Date: Fri, 28 Aug 2026 10:29:09 +0200 Subject: [PATCH 02/99] fix(test-optimization): restore pre-defer suite reporting (#10017) --- .../mocha-reporter-exits-after-suite.js | 10 - integration-tests/cucumber/cucumber.spec.js | 12 +- .../cypress-reporting-instrumentation.spec.js | 44 ++- integration-tests/jest/jest.core.spec.js | 3 +- integration-tests/mocha/mocha.spec.js | 60 +--- integration-tests/vitest/vitest.core.spec.js | 6 +- packages/datadog-instrumentations/src/jest.js | 4 +- .../src/vitest-main.js | 1 - packages/datadog-plugin-cucumber/src/index.js | 2 - .../src/cypress-plugin.js | 18 +- packages/datadog-plugin-jest/src/index.js | 6 - packages/datadog-plugin-mocha/src/index.js | 50 +-- .../datadog-plugin-playwright/src/index.js | 2 - packages/datadog-plugin-vitest/src/index.js | 6 - .../exporters/agent-proxy/index.js | 7 - .../exporters/ci-visibility-exporter.js | 158 +-------- packages/dd-trace/src/plugins/ci_plugin.js | 7 +- .../test/ci-visibility/ci-plugin.spec.js | 15 +- .../exporters/agent-proxy/agent-proxy.spec.js | 36 +- .../exporters/ci-visibility-exporter.spec.js | 330 ++---------------- 20 files changed, 86 insertions(+), 691 deletions(-) delete mode 100644 integration-tests/ci-visibility/mocha-reporter-exits-after-suite.js diff --git a/integration-tests/ci-visibility/mocha-reporter-exits-after-suite.js b/integration-tests/ci-visibility/mocha-reporter-exits-after-suite.js deleted file mode 100644 index 64c9e9f05ff..00000000000 --- a/integration-tests/ci-visibility/mocha-reporter-exits-after-suite.js +++ /dev/null @@ -1,10 +0,0 @@ -'use strict' - -module.exports = function ExitingReporter (runner) { - runner.on('suite end', (suite) => { - if (suite.title !== 'mocha-test-pass-two') return - - const exporter = require('dd-trace')._tracer._exporter - exporter.flush(() => process.exit(0)) - }) -} diff --git a/integration-tests/cucumber/cucumber.spec.js b/integration-tests/cucumber/cucumber.spec.js index 326db4fb680..d3ac585d831 100644 --- a/integration-tests/cucumber/cucumber.spec.js +++ b/integration-tests/cucumber/cucumber.spec.js @@ -918,7 +918,7 @@ describe(`cucumber@${version} commonJS`, () => { } const eventTypes = eventsRequest.payload.events.map(event => event.type) - assertObjectContains(eventTypes, ['test', 'test_suite_end', 'test_session_end', 'test_module_end']) + assertObjectContains(eventTypes, ['test', 'test_session_end', 'test_module_end', 'test_suite_end']) const numSuites = eventTypes.reduce( (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 ) @@ -992,7 +992,7 @@ describe(`cucumber@${version} commonJS`, () => { assert.ok(testSession.metrics[TEST_CODE_COVERAGE_LINES_PCT]) const eventTypes = eventsRequest.payload.events.map(event => event.type) - assertObjectContains(eventTypes, ['test', 'test_suite_end', 'test_session_end', 'test_module_end']) + assertObjectContains(eventTypes, ['test', 'test_session_end', 'test_module_end', 'test_suite_end']) const numSuites = eventTypes.reduce( (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 ) @@ -1033,7 +1033,7 @@ describe(`cucumber@${version} commonJS`, () => { receiver.assertPayloadReceived(({ payload }) => { const eventTypes = payload.events.map(event => event.type) - assertObjectContains(eventTypes, ['test', 'test_suite_end', 'test_session_end', 'test_module_end']) + assertObjectContains(eventTypes, ['test', 'test_session_end', 'test_module_end', 'test_suite_end']) const testSession = payload.events.find(event => event.type === 'test_session_end').content assert.strictEqual(testSession.meta[TEST_ITR_TESTS_SKIPPED], 'false') assert.strictEqual(testSession.meta[TEST_CODE_COVERAGE_ENABLED], 'false') @@ -1098,7 +1098,7 @@ describe(`cucumber@${version} commonJS`, () => { assert.strictEqual(skippedSuite.meta[TEST_STATUS], 'skip') assert.strictEqual(skippedSuite.meta[TEST_SKIPPED_BY_ITR], 'true') - assertObjectContains(eventTypes, ['test', 'test_suite_end', 'test_session_end', 'test_module_end']) + assertObjectContains(eventTypes, ['test', 'test_session_end', 'test_module_end', 'test_suite_end']) const numSuites = eventTypes.reduce( (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 ) @@ -1191,7 +1191,7 @@ describe(`cucumber@${version} commonJS`, () => { receiver.assertPayloadReceived(({ payload }) => { const eventTypes = payload.events.map(event => event.type) // because they are not skipped - assertObjectContains(eventTypes, ['test', 'test_suite_end', 'test_session_end', 'test_module_end']) + assertObjectContains(eventTypes, ['test', 'test_session_end', 'test_module_end', 'test_suite_end']) const numSuites = eventTypes.reduce( (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 ) @@ -1238,7 +1238,7 @@ describe(`cucumber@${version} commonJS`, () => { receiver.assertPayloadReceived(({ payload }) => { const eventTypes = payload.events.map(event => event.type) // because they are not skipped - assertObjectContains(eventTypes, ['test', 'test_suite_end', 'test_session_end', 'test_module_end']) + assertObjectContains(eventTypes, ['test', 'test_session_end', 'test_module_end', 'test_suite_end']) const numSuites = eventTypes.reduce( (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 ) diff --git a/integration-tests/cypress/cypress-reporting-instrumentation.spec.js b/integration-tests/cypress/cypress-reporting-instrumentation.spec.js index 72f490c77c8..6ce84a415fa 100644 --- a/integration-tests/cypress/cypress-reporting-instrumentation.spec.js +++ b/integration-tests/cypress/cypress-reporting-instrumentation.spec.js @@ -549,13 +549,17 @@ moduleTypes.forEach(({ ({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { const events = payloads.flatMap(({ payload }) => payload.events) - for (const eventType of ['test_session_end', 'test_module_end', 'test_suite_end']) { + for (const eventType of ['test_session_end', 'test_module_end']) { const event = events.find(event => event.type === eventType) assert.ok(event, `expected ${eventType} event`) assert.strictEqual(event.content.meta[TEST_STATUS], 'fail') assert.strictEqual(event.content.error, 1) assert.match(event.content.meta[ERROR_MESSAGE], expectedError) } + const testSuite = events.find(event => event.type === 'test_suite_end') + assert.ok(testSuite, 'expected test_suite_end event') + assert.strictEqual(testSuite.content.meta[TEST_STATUS], 'pass') + assert.strictEqual(testSuite.content.error, 0) }, { hardTimeout: 60000 } ) @@ -929,13 +933,17 @@ moduleTypes.forEach(({ ({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { const events = payloads.flatMap(({ payload }) => payload.events) - for (const eventType of ['test_session_end', 'test_module_end', 'test_suite_end']) { + for (const eventType of ['test_session_end', 'test_module_end']) { const event = events.find(event => event.type === eventType) assert.ok(event, `expected ${eventType} event`) assert.strictEqual(event.content.meta[TEST_STATUS], 'fail') assert.strictEqual(event.content.error, 1) assert.match(event.content.meta[ERROR_MESSAGE], /manual after:run failed after Datadog/) } + const testSuite = events.find(event => event.type === 'test_suite_end') + assert.ok(testSuite, 'expected test_suite_end event') + assert.strictEqual(testSuite.content.meta[TEST_STATUS], 'pass') + assert.strictEqual(testSuite.content.error, 0) }, { hardTimeout: 60000 } ) @@ -1130,13 +1138,21 @@ moduleTypes.forEach(({ ({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { const events = payloads.flatMap(({ payload }) => payload.events) - for (const eventType of ['test_session_end', 'test_module_end', 'test_suite_end']) { + const hierarchyEventTypes = ['test_session_end', 'test_module_end'] + if (lifecycle === 'afterSpec') hierarchyEventTypes.push('test_suite_end') + for (const eventType of hierarchyEventTypes) { const testSessionTraceEvents = events.filter(event => event.type === eventType) assert.strictEqual(testSessionTraceEvents.length, 1, `expected one ${eventType} event`) assert.strictEqual(testSessionTraceEvents[0].content.meta[TEST_STATUS], 'fail') assert.strictEqual(testSessionTraceEvents[0].content.error, 1) assert.match(testSessionTraceEvents[0].content.meta[ERROR_MESSAGE], new RegExp(errorMessage)) } + if (lifecycle === 'afterRun') { + const testSuite = events.find(event => event.type === 'test_suite_end') + assert.ok(testSuite, 'expected test_suite_end event') + assert.strictEqual(testSuite.content.meta[TEST_STATUS], 'pass') + assert.strictEqual(testSuite.content.error, 0) + } const testEvent = events.find(event => event.type === 'test' && @@ -2097,7 +2113,6 @@ moduleTypes.forEach(({ finishedTestsByFile: cypressPlugin.finishedTestsByFile, testsToSkip: cypressPlugin.testsToSkip, testSuiteSpan: cypressPlugin.testSuiteSpan, - finishedTestSuiteSpans: cypressPlugin.finishedTestSuiteSpans, } afterEach(() => { @@ -2110,7 +2125,6 @@ moduleTypes.forEach(({ cypressPlugin.finishedTestsByFile = originalState.finishedTestsByFile cypressPlugin.testsToSkip = originalState.testsToSkip cypressPlugin.testSuiteSpan = originalState.testSuiteSpan - cypressPlugin.finishedTestSuiteSpans = originalState.finishedTestSuiteSpans sinon.restore() }) @@ -2145,26 +2159,25 @@ moduleTypes.forEach(({ sinon.assert.calledOnceWithExactly(init, tracer, cypressConfig) }) - for (const [description, cypressConfig, shouldDefer] of [ - ['does not retain completed suites without an after:run boundary', { + for (const [description, cypressConfig] of [ + ['finishes completed suites without an after:run boundary', { isTextTerminal: false, isInteractive: true, experimentalInteractiveRunEvents: false, - }, false], - ['retains completed suites in terminal runs', { + }], + ['finishes completed suites in terminal runs', { isTextTerminal: true, // Cypress 12 can leave this true during `cypress run`. isInteractive: true, experimentalInteractiveRunEvents: false, - }, true], - ['retains completed suites when interactive run events are enabled', { + }], + ['finishes completed suites when interactive run events are enabled', { isTextTerminal: false, isInteractive: true, experimentalInteractiveRunEvents: true, - }, true], + }], ]) { it(description, () => { - const deferTestSuiteSpan = sinon.stub() const testSuiteSpan = { finish: sinon.stub(), setTag: sinon.stub(), @@ -2173,15 +2186,12 @@ moduleTypes.forEach(({ cypressPlugin.finishedTestsByFile = {} cypressPlugin.testsToSkip = [] cypressPlugin.testSuiteSpan = testSuiteSpan - cypressPlugin.finishedTestSuiteSpans = [] - cypressPlugin.tracer = { _tracer: { _exporter: { deferTestSuiteSpan } } } + cypressPlugin.tracer = { _tracer: { _exporter: {} } } sinon.stub(cypressPlugin, 'ciVisEvent') cypressPlugin.afterSpec({ relative: 'cypress/e2e/basic-pass.js' }, { stats: { tests: 1 } }) sinon.assert.calledOnce(testSuiteSpan.finish) - assert.strictEqual(deferTestSuiteSpan.calledOnceWithExactly(testSuiteSpan), shouldDefer) - assert.strictEqual(cypressPlugin.finishedTestSuiteSpans.length, shouldDefer ? 1 : 0) }) } diff --git a/integration-tests/jest/jest.core.spec.js b/integration-tests/jest/jest.core.spec.js index 92d1d76afc3..c5e4761d80f 100644 --- a/integration-tests/jest/jest.core.spec.js +++ b/integration-tests/jest/jest.core.spec.js @@ -2259,11 +2259,12 @@ describe(`jest@${JEST_VERSION} commonJS`, () => { assert.strictEqual(tests.length, 1) const [testSuite] = testSuites - for (const event of [testSession, testModule, testSuite]) { + for (const event of [testSession, testModule]) { assert.strictEqual(event.meta[TEST_STATUS], 'fail') assert.strictEqual(event.error, 1) assert.match(event.meta[ERROR_MESSAGE], /custom reporter failed/) } + assert.strictEqual(testSuite.meta[TEST_STATUS], 'pass') assert.strictEqual(testSuite.test_session_id.toString(), testSession.test_session_id.toString()) assert.strictEqual(testSuite.test_module_id.toString(), testModule.test_module_id.toString()) assert.strictEqual(tests[0].test_suite_id.toString(), testSuite.test_suite_id.toString()) diff --git a/integration-tests/mocha/mocha.spec.js b/integration-tests/mocha/mocha.spec.js index 1d4f502f334..a28efcae478 100644 --- a/integration-tests/mocha/mocha.spec.js +++ b/integration-tests/mocha/mocha.spec.js @@ -1216,54 +1216,6 @@ describe(`mocha@${MOCHA_VERSION}`, function () { assert.strictEqual((testOutput.match(/MOCHA REUSABLE TEST EXECUTED/g) || []).length, 4) }) - it('reports a completed suite when the process exits before session finalization', async function () { - this.timeout(20_000) - const startedAt = Date.now() - childProcess = exec( - [ - 'node node_modules/mocha/bin/mocha', - './ci-visibility/mocha-plugin-tests/passing.js', - '--reporter ./ci-visibility/mocha-reporter-exits-after-suite.js', - ].join(' '), - { - cwd, - env: { - ...getCiVisAgentlessConfig(receiver.port), - }, - } - ) - - const eventsPromise = receiver.gatherPayloadsUntilChildExit( - childProcess, - ({ url }) => url.endsWith('/api/v2/citestcycle'), - (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - const suiteEvents = events.filter(event => - event.type === 'test_suite_end' && - event.content.meta[TEST_SUITE] === 'ci-visibility/mocha-plugin-tests/passing.js' - ) - assert.strictEqual(suiteEvents.length, 1) - assert.strictEqual(suiteEvents[0].content.meta[TEST_STATUS], 'pass') - assert.strictEqual(suiteEvents[0].content.error, 0) - - const testEvent = events.find(event => - event.type === 'test' && event.content.meta[TEST_NAME] === 'mocha-test-pass-two can pass' - ) - assert.ok(testEvent, 'expected completed test event') - assert.strictEqual(testEvent.content.meta[TEST_STATUS], 'pass') - }, - { hardTimeout: 20_000 } - ) - - const [[exitCode]] = await Promise.all([ - once(childProcess, 'exit'), - eventsPromise, - ]) - - assert.strictEqual(exitCode, 0) - assert.ok(Date.now() - startedAt < 12_000, 'final writer flush should remain bounded') - }) - it('can run tests and report tests with the APM protocol (old agents)', (done) => { receiver.setInfoResponse({ endpoints: [] }) receiver.payloadReceived(({ url }) => url === '/v0.4/traces').then(({ payload }) => { @@ -3064,7 +3016,7 @@ describe(`mocha@${MOCHA_VERSION}`, function () { assert.strictEqual(packfileRequest.headers['dd-api-key'], '1') const eventTypes = eventsRequest.payload.events.map(event => event.type) - assertObjectContains(eventTypes, ['test', 'test_suite_end', 'test_session_end', 'test_module_end']) + assertObjectContains(eventTypes, ['test', 'test_session_end', 'test_module_end', 'test_suite_end']) const numSuites = eventTypes.reduce( (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 ) @@ -3179,7 +3131,7 @@ describe(`mocha@${MOCHA_VERSION}`, function () { assert.ok(testSession.metrics[TEST_CODE_COVERAGE_LINES_PCT]) const eventTypes = eventsRequest.payload.events.map(event => event.type) - assertObjectContains(eventTypes, ['test', 'test_suite_end', 'test_session_end', 'test_module_end']) + assertObjectContains(eventTypes, ['test', 'test_session_end', 'test_module_end', 'test_suite_end']) const numSuites = eventTypes.reduce( (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 ) @@ -3222,7 +3174,7 @@ describe(`mocha@${MOCHA_VERSION}`, function () { receiver.assertPayloadReceived(({ headers, payload }) => { assert.strictEqual(headers['dd-api-key'], '1') const eventTypes = payload.events.map(event => event.type) - assertObjectContains(eventTypes, ['test', 'test_suite_end', 'test_session_end', 'test_module_end']) + assertObjectContains(eventTypes, ['test', 'test_session_end', 'test_module_end', 'test_suite_end']) const testSession = payload.events.find(event => event.type === 'test_session_end').content assert.strictEqual(testSession.meta[TEST_ITR_TESTS_SKIPPED], 'false') assert.strictEqual(testSession.meta[TEST_CODE_COVERAGE_ENABLED], 'false') @@ -3281,7 +3233,7 @@ describe(`mocha@${MOCHA_VERSION}`, function () { assert.strictEqual(skippedSuite.meta[TEST_STATUS], 'skip') assert.strictEqual(skippedSuite.meta[TEST_SKIPPED_BY_ITR], 'true') - assertObjectContains(eventTypes, ['test', 'test_suite_end', 'test_session_end', 'test_module_end']) + assertObjectContains(eventTypes, ['test', 'test_session_end', 'test_module_end', 'test_suite_end']) const numSuites = eventTypes.reduce( (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 ) @@ -3566,7 +3518,7 @@ describe(`mocha@${MOCHA_VERSION}`, function () { assert.strictEqual(headers['dd-api-key'], '1') const eventTypes = payload.events.map(event => event.type) // because they are not skipped - assertObjectContains(eventTypes, ['test', 'test_suite_end', 'test_session_end', 'test_module_end']) + assertObjectContains(eventTypes, ['test', 'test_session_end', 'test_module_end', 'test_suite_end']) const numSuites = eventTypes.reduce( (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 ) @@ -3614,7 +3566,7 @@ describe(`mocha@${MOCHA_VERSION}`, function () { assert.strictEqual(headers['dd-api-key'], '1') const eventTypes = payload.events.map(event => event.type) // because they are not skipped - assertObjectContains(eventTypes, ['test', 'test_suite_end', 'test_session_end', 'test_module_end']) + assertObjectContains(eventTypes, ['test', 'test_session_end', 'test_module_end', 'test_suite_end']) const numSuites = eventTypes.reduce( (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 ) diff --git a/integration-tests/vitest/vitest.core.spec.js b/integration-tests/vitest/vitest.core.spec.js index 313dd21ac06..8bfc6e1fb48 100644 --- a/integration-tests/vitest/vitest.core.spec.js +++ b/integration-tests/vitest/vitest.core.spec.js @@ -170,11 +170,12 @@ versions.forEach((version) => { const { testSession, testModule, testSuite, tests } = assertCompleteTestSessionTrace(events, testOutput) assert.strictEqual(events.filter(event => event.type === 'test_suite_end').length, 1) - for (const event of [testSession, testModule, testSuite]) { + for (const event of [testSession, testModule]) { assert.strictEqual(event.meta[TEST_STATUS], 'fail') assert.strictEqual(event.error, 1) assert.match(event.meta[ERROR_MESSAGE], /custom Vitest reporter failed/) } + assert.strictEqual(testSuite.meta[TEST_STATUS], 'pass') assert.deepStrictEqual( [...new Set(tests.map(test => test.meta[TEST_STATUS]))].sort(), ['pass', 'skip'] @@ -214,11 +215,12 @@ versions.forEach((version) => { const { testSession, testModule, testSuite } = assertCompleteTestSessionTrace(events, testOutput) assert.strictEqual(events.filter(event => event.type === 'test_suite_end').length, 1) - for (const event of [testSession, testModule, testSuite]) { + for (const event of [testSession, testModule]) { assert.strictEqual(event.meta[TEST_STATUS], 'fail') assert.strictEqual(event.error, 1) assert.match(event.meta[ERROR_MESSAGE], /custom Vitest reporter failed/) } + assert.strictEqual(testSuite.meta[TEST_STATUS], 'pass') }, { hardTimeout: 20_000 } ) diff --git a/packages/datadog-instrumentations/src/jest.js b/packages/datadog-instrumentations/src/jest.js index a9c070cdf9d..da31299ec5e 100644 --- a/packages/datadog-instrumentations/src/jest.js +++ b/packages/datadog-instrumentations/src/jest.js @@ -3067,9 +3067,7 @@ function getCliWrapper (isNewJestVersion) { result = await runCLI.apply(this, arguments) } catch (error) { try { - await waitForTestSessionFinish(getTestSessionFinishPayload('fail', error, { - isTestSessionFinalizationError: true, - })) + await waitForTestSessionFinish(getTestSessionFinishPayload('fail', error)) } catch (finalizationError) { log.error('Jest test session finalization error: %s', finalizationError) } diff --git a/packages/datadog-instrumentations/src/vitest-main.js b/packages/datadog-instrumentations/src/vitest-main.js index 1af99bdf1c8..f2f2ad79c54 100644 --- a/packages/datadog-instrumentations/src/vitest-main.js +++ b/packages/datadog-instrumentations/src/vitest-main.js @@ -1207,7 +1207,6 @@ function getFinishWrapper (exitOrClose) { status: runError ? 'fail' : (areAllSuitesSkipped ? 'skip' : getSessionStatus(this.state)), testCodeCoverageLinesTotal, error, - isTestSessionFinalizationError: Boolean(runError), isEarlyFlakeDetectionEnabled, isEarlyFlakeDetectionFaulty, isTestManagementTestsEnabled, diff --git a/packages/datadog-plugin-cucumber/src/index.js b/packages/datadog-plugin-cucumber/src/index.js index 3512df1862f..ece96c42dad 100644 --- a/packages/datadog-plugin-cucumber/src/index.js +++ b/packages/datadog-plugin-cucumber/src/index.js @@ -124,7 +124,6 @@ class CucumberPlugin extends CiPlugin { this.testSessionSpan.setTag('error', error) this.testModuleSpan.setTag('error', error) } - this.tracer._exporter.exportDeferredTestSuiteSpans?.() this.testModuleSpan.finish() this.telemetry.ciVisEvent(TELEMETRY_EVENT_FINISHED, 'module') this.testSessionSpan.finish() @@ -202,7 +201,6 @@ class CucumberPlugin extends CiPlugin { this.addSub('ci:cucumber:test-suite:finish', ({ status, testSuitePath }) => { const testSuiteSpan = this._testSuiteSpansByTestSuite.get(testSuitePath) testSuiteSpan.setTag(TEST_STATUS, status) - this.tracer._exporter.deferTestSuiteSpan?.(testSuiteSpan) testSuiteSpan.finish() this.telemetry.ciVisEvent(TELEMETRY_EVENT_FINISHED, 'suite') }) diff --git a/packages/datadog-plugin-cypress/src/cypress-plugin.js b/packages/datadog-plugin-cypress/src/cypress-plugin.js index 13585663d58..1cdffd58b64 100644 --- a/packages/datadog-plugin-cypress/src/cypress-plugin.js +++ b/packages/datadog-plugin-cypress/src/cypress-plugin.js @@ -587,7 +587,6 @@ class CypressPlugin { this.pendingScreenshotUploads = [] this.activeTestSpan = null this.testSuiteSpan = null - this.finishedTestSuiteSpans = [] this.testModuleSpan = null this.testSessionSpan = null this.command = undefined @@ -1272,7 +1271,7 @@ class CypressPlugin { return details } - afterRun (suiteStats, error, shouldFailFinishedSuites = true) { + afterRun (suiteStats, error) { if (!this._isInit) { log.warn('Attemping to call afterRun without initializating the plugin first') return @@ -1284,13 +1283,6 @@ class CypressPlugin { this.testModuleSpan.setTag(TEST_STATUS, testStatus) this.testSessionSpan.setTag(TEST_STATUS, testStatus) - for (const span of this.finishedTestSuiteSpans) { - if (error && shouldFailFinishedSuites) { - span.setTag(TEST_STATUS, 'fail') - span.setTag('error', error) - } - } - this.finishedTestSuiteSpans = [] if (error) { this.testModuleSpan.setTag('error', error) this.testSessionSpan.setTag('error', error) @@ -1332,7 +1324,6 @@ class CypressPlugin { }) finishAllTraceSpans(this.testSessionSpan) - this.tracer._tracer._exporter?.exportDeferredTestSuiteSpans?.() } return new Promise(resolve => { @@ -1627,12 +1618,7 @@ class CypressPlugin { if (error || latestError) { this.testSuiteSpan.setTag('error', error || latestError) } - const canRunAfterRun = this.cypressConfig.isTextTerminal || - this.cypressConfig.experimentalInteractiveRunEvents - const exporter = this.tracer._tracer._exporter - if (canRunAfterRun && exporter?.deferTestSuiteSpan) exporter.deferTestSuiteSpan(this.testSuiteSpan) this.testSuiteSpan.finish() - if (canRunAfterRun) this.finishedTestSuiteSpans.push(this.testSuiteSpan) this.testSuiteSpan = null this.ciVisEvent(TELEMETRY_EVENT_FINISHED, 'suite') } @@ -1650,7 +1636,7 @@ class CypressPlugin { if (error) { this.abortPendingScreenshotUploads(error) - return this.afterRun(undefined, error, false) + return this.afterRun(undefined, error) } const screenshotUploadsPromise = waitForScreenshotUploads() diff --git a/packages/datadog-plugin-jest/src/index.js b/packages/datadog-plugin-jest/src/index.js index 69154484ce0..7e9513b18ab 100644 --- a/packages/datadog-plugin-jest/src/index.js +++ b/packages/datadog-plugin-jest/src/index.js @@ -113,7 +113,6 @@ class JestPlugin extends CiPlugin { hasUnskippableSuites, hasForcedToRunSuites, error, - isTestSessionFinalizationError, isEarlyFlakeDetectionEnabled, isEarlyFlakeDetectionFaulty, isTestManagementTestsEnabled, @@ -124,9 +123,6 @@ class JestPlugin extends CiPlugin { this.testModuleSpan.setTag(TEST_STATUS, status) if (error) { - if (isTestSessionFinalizationError) { - this.tracer._exporter.setDeferredTestSuiteError?.(error) - } this.testSessionSpan.setTag('error', error) this.testModuleSpan.setTag('error', error) } @@ -163,7 +159,6 @@ class JestPlugin extends CiPlugin { this.testSessionSpan.setTag(TEST_MANAGEMENT_ENABLED, 'true') } - this.tracer._exporter.exportDeferredTestSuiteSpans?.() this.testModuleSpan.finish() this.telemetry.ciVisEvent(TELEMETRY_EVENT_FINISHED, 'module') this.testSessionSpan.finish() @@ -334,7 +329,6 @@ class JestPlugin extends CiPlugin { this.pendingTestSuiteFinishes.add(pendingFinish) const finish = () => { - this.tracer._exporter.deferTestSuiteSpan?.(testSuiteSpan) testSuiteSpan.finish() this.telemetry.ciVisEvent(TELEMETRY_EVENT_FINISHED, 'suite') // Suites potentially run in a different process than the session, diff --git a/packages/datadog-plugin-mocha/src/index.js b/packages/datadog-plugin-mocha/src/index.js index b2a7ada8ac8..2185fbfa88f 100644 --- a/packages/datadog-plugin-mocha/src/index.js +++ b/packages/datadog-plugin-mocha/src/index.js @@ -1,7 +1,6 @@ 'use strict' const { performance } = require('node:perf_hooks') -const dateNow = Date.now const { fileURLToPath } = require('node:url') const { channel } = require('dc-polyfill') @@ -174,15 +173,6 @@ class MochaPlugin extends CiPlugin { this._testTitleToParams = {} this.sourceRoot = process.cwd() - this._pendingTestSuiteSpans = [] - this._timeOrigin = dateNow() - this._perfOrigin = performance.now() - - this.addSub('ci:mocha:session:start', () => { - this._pendingTestSuiteSpans = [] - this._timeOrigin = dateNow() - this._perfOrigin = performance.now() - }) this.addSub('ci:mocha:worker:configuration', ({ libraryConfig, @@ -482,19 +472,7 @@ class MochaPlugin extends CiPlugin { if (!testSuiteSpan.context().getTag(TEST_STATUS)) { testSuiteSpan.setTag(TEST_STATUS, status) } - const exporter = this.tracer._exporter - if (exporter.deferTestSuiteSpan) { - exporter.deferTestSuiteSpan(testSuiteSpan) - testSuiteSpan.finish() - } else if (exporter.exportDeferredTestSuiteSpans) { - const finishTime = this._now() - this._pendingTestSuiteSpans.push({ - span: testSuiteSpan, - finishTime, - }) - } else { - testSuiteSpan.finish() - } + testSuiteSpan.finish() this.telemetry.ciVisEvent(TELEMETRY_EVENT_FINISHED, 'suite') } }) @@ -544,7 +522,6 @@ class MochaPlugin extends CiPlugin { }) this.addSub('ci:mocha:worker:finish', ({ onDone } = {}) => { - this._finishPendingTestSuiteSpans() this.tracer._exporter.flush(onDone) }) @@ -755,7 +732,6 @@ class MochaPlugin extends CiPlugin { this.testSessionSpan.setTag('error', error) this.testModuleSpan.setTag('error', error) if (isFrameworkError) { - this.tracer._exporter.setDeferredTestSuiteError?.(error) for (const testSuiteSpan of this._testSuiteSpansByTestSuite.values()) { testSuiteSpan.setTag(TEST_STATUS, 'fail') testSuiteSpan.setTag('error', error) @@ -763,8 +739,6 @@ class MochaPlugin extends CiPlugin { } } - this._finishPendingTestSuiteSpans() - if (isParallel) { this.testSessionSpan.setTag(MOCHA_IS_PARALLEL, 'true') } @@ -802,7 +776,6 @@ class MochaPlugin extends CiPlugin { this.testSessionSpan.setTag(TEST_EARLY_FLAKE_ABORT_REASON, 'faulty') } - this.tracer._exporter.exportDeferredTestSuiteSpans?.() this.testModuleSpan.finish() this.telemetry.ciVisEvent(TELEMETRY_EVENT_FINISHED, 'module') this.testSessionSpan.finish() @@ -824,27 +797,6 @@ class MochaPlugin extends CiPlugin { }) } - /** - * Returns the current time in the coordinate system used by spans. - * - * @returns {number} - */ - _now () { - return this._timeOrigin + performance.now() - this._perfOrigin - } - - /** - * Finishes suites retained for a later framework finalization event. - * - * @returns {void} - */ - _finishPendingTestSuiteSpans () { - for (const { span, finishTime } of this._pendingTestSuiteSpans) { - span.finish(finishTime) - } - this._pendingTestSuiteSpans = [] - } - /** * Starts a Jasmine test span around WebdriverIO's test-function wrapper. * diff --git a/packages/datadog-plugin-playwright/src/index.js b/packages/datadog-plugin-playwright/src/index.js index 6b6b7420409..ee7281d948c 100644 --- a/packages/datadog-plugin-playwright/src/index.js +++ b/packages/datadog-plugin-playwright/src/index.js @@ -173,7 +173,6 @@ class PlaywrightPlugin extends CiPlugin { this.testSessionSpan.setTag(TEST_MANAGEMENT_ENABLED, 'true') } - this.tracer._exporter.exportDeferredTestSuiteSpans?.() this.testModuleSpan.finish() this.telemetry.ciVisEvent(TELEMETRY_EVENT_FINISHED, 'module') this.testSessionSpan.finish() @@ -255,7 +254,6 @@ class PlaywrightPlugin extends CiPlugin { this.numFailedSuites++ } - this.tracer._exporter.deferTestSuiteSpan?.(testSuiteSpan) testSuiteSpan.finish() this.telemetry.ciVisEvent(TELEMETRY_EVENT_FINISHED, 'suite') }) diff --git a/packages/datadog-plugin-vitest/src/index.js b/packages/datadog-plugin-vitest/src/index.js index dde64840479..767910ff52b 100644 --- a/packages/datadog-plugin-vitest/src/index.js +++ b/packages/datadog-plugin-vitest/src/index.js @@ -523,7 +523,6 @@ class VitestPlugin extends CiPlugin { this.telemetry.ciVisEvent(TELEMETRY_CODE_COVERAGE_FINISHED, 'suite', { library: coverageLibrary }) this.telemetry.distribution(TELEMETRY_CODE_COVERAGE_NUM_FILES, {}, relativeFiles.length) } - this.tracer._exporter.deferTestSuiteSpan?.(testSuiteSpan) testSuiteSpan.finish() finishAllTraceSpans(testSuiteSpan) } @@ -567,7 +566,6 @@ class VitestPlugin extends CiPlugin { this.addSub('ci:vitest:session:finish', ({ status, error, - isTestSessionFinalizationError, testCodeCoverageLinesTotal, isEarlyFlakeDetectionEnabled, isEarlyFlakeDetectionFaulty, @@ -590,9 +588,6 @@ class VitestPlugin extends CiPlugin { this.testSessionSpan.setTag(TEST_STATUS, status) this.testModuleSpan.setTag(TEST_STATUS, status) if (error) { - if (isTestSessionFinalizationError) { - this.tracer._exporter.setDeferredTestSuiteError?.(error) - } this.testModuleSpan.setTag('error', error) this.testSessionSpan.setTag('error', error) } @@ -622,7 +617,6 @@ class VitestPlugin extends CiPlugin { if (vitestPool) { this.testSessionSpan.setTag(VITEST_POOL, vitestPool) } - this.tracer._exporter.exportDeferredTestSuiteSpans?.() this.testModuleSpan.finish() this.telemetry.ciVisEvent(TELEMETRY_EVENT_FINISHED, 'module') this.testSessionSpan.finish() diff --git a/packages/dd-trace/src/ci-visibility/exporters/agent-proxy/index.js b/packages/dd-trace/src/ci-visibility/exporters/agent-proxy/index.js index b2888103bc8..8a900a27c2d 100644 --- a/packages/dd-trace/src/ci-visibility/exporters/agent-proxy/index.js +++ b/packages/dd-trace/src/ci-visibility/exporters/agent-proxy/index.js @@ -54,7 +54,6 @@ class AgentProxyCiVisibilityExporter extends CiVisibilityExporter { } fetchAgentInfo(this._url, (err, agentInfo) => { - const initializationFinalFlush = this._initializationRequest?.finalFlush this._initializationRequest = undefined const initializationAborted = initializationController.signal.aborted const agentInfoError = err || (initializationAborted ? initializationController.signal.reason : undefined) @@ -112,14 +111,8 @@ class AgentProxyCiVisibilityExporter extends CiVisibilityExporter { this._resolveCanUseCiVisProtocol(isEvpCompatible) if (initializationAborted) { this.resetUncodedTraces() - this.resetDeferredTestSuiteSpans() return } - if (isEvpCompatible) { - if (initializationFinalFlush) this.exportDeferredTestSuiteSpans() - } else { - this.resetDeferredTestSuiteSpans() - } this.exportUncodedTraces() this.exportUncodedCoverages() }, initializationOptions, request) diff --git a/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js b/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js index 7f1a7ed5dd2..50185110255 100644 --- a/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js +++ b/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js @@ -15,16 +15,13 @@ const { getTestManagementTests: getTestManagementTestsRequest } = const { writeSettingsToCache } = require('../test-optimization-cache') const { CACHE_MISS, TestOptimizationHttpCache } = require('../test-optimization-http-cache') const { MAX_RETRIES } = require('../test-optimization-http-cache-schema') -const { incrementCountMetric, TELEMETRY_EVENTS_ENQUEUED_FOR_SERIALIZATION } = require('../telemetry') const { uploadCoverageReport: uploadCoverageReportRequest } = require('../requests/upload-coverage-report') const { uploadTestScreenshot: uploadTestScreenshotRequest } = require('../requests/upload-test-screenshot') const { parsers } = require('../../config/parsers') const log = require('../../log') -const spanFormat = require('../../span_format') const { getSegment } = require('../../util') const BufferingExporter = require('../../exporters/common/buffering-exporter') const { GIT_REPOSITORY_URL, GIT_COMMIT_SHA } = require('../../plugins/util/tags') -const { TEST_STATUS } = require('../../plugins/util/test') const { createFinalFlushTimeoutError, FINAL_FLUSH_FALLBACK_DELAY, @@ -74,10 +71,12 @@ function getTestConfigurationTags (tags) { }, {}) } +function isTestSessionEvent (span) { + return span.type === 'test_session_end' || span.type === 'test_suite_end' || span.type === 'test_module_end' +} + function getIsTestSessionTrace (trace) { - return trace.some(span => - span.type === 'test_session_end' || span.type === 'test_suite_end' || span.type === 'test_module_end' - ) + return trace.some(isTestSessionEvent) } /** @@ -191,7 +190,6 @@ class CiVisibilityExporter extends BufferingExporter { #deferredTestSessionTraces = [] #pendingScreenshotUploads = new Set() #screenshotFlushWaiters = new Set() - #deferredTestSuiteSpans = new Map() #gitUploadTimeoutId constructor (config, options = {}) { @@ -590,34 +588,7 @@ class CiVisibilityExporter extends BufferingExporter { export (trace) { this.#resetFinalFlush() - - if (this.#deferredTestSuiteSpans.size === 0) { - this.#exportTrace(trace) - return - } - - let hasDeferredTestSuiteSpan = false - let immediateTrace - for (let index = 0; index < trace.length; index++) { - const formattedSpan = trace[index] - const spanId = formattedSpan.span_id?.toString() - const deferredTestSuiteSpan = this.#deferredTestSuiteSpans.get(spanId) - if (deferredTestSuiteSpan) { - if (!hasDeferredTestSuiteSpan && index > 0) immediateTrace = trace.slice(0, index) - hasDeferredTestSuiteSpan = true - if (deferredTestSuiteSpan.emitted || (this._isInitialized && !this.canReportSessionTraces())) { - this.#deferredTestSuiteSpans.delete(spanId) - } else { - deferredTestSuiteSpan.formattedSpan = formattedSpan - } - } else if (hasDeferredTestSuiteSpan) { - immediateTrace ??= [] - immediateTrace.push(formattedSpan) - } - } - - if (!hasDeferredTestSuiteSpan) this.#exportTrace(trace) - else if (immediateTrace) this.#exportTrace(immediateTrace) + this.#exportTrace(trace) } /** @@ -634,6 +605,11 @@ class CiVisibilityExporter extends BufferingExporter { } const isTestSessionTrace = getIsTestSessionTrace(trace) if (!this.canReportSessionTraces() && isTestSessionTrace) { + const testTrace = [] + for (const span of trace) { + if (!isTestSessionEvent(span)) testTrace.push(span) + } + if (testTrace.length > 0) this._export(testTrace) return } if (this._export(trace, undefined, undefined, isTestSessionTrace) === false && isTestSessionTrace) { @@ -641,115 +617,6 @@ class CiVisibilityExporter extends BufferingExporter { } } - /** - * Retains a completed suite until finalization so a later framework error can still update it. - * - * @param {import('../../opentracing/span')} testSuiteSpan - * @returns {void} - */ - deferTestSuiteSpan (testSuiteSpan) { - this.#resetFinalFlush() - this.#deferredTestSuiteSpans.set(testSuiteSpan.context()._spanId.toString(), { - span: testSuiteSpan, - formattedSpan: undefined, - emitted: false, - }) - } - - /** - * Retains formatted test suite events received from test framework workers until finalization. - * - * @param {Array} trace - * @returns {void} - */ - exportTraceWithDeferredTestSuite (trace) { - for (const formattedSpan of trace) { - if (formattedSpan.type !== 'test_suite_end') continue - - this.#deferredTestSuiteSpans.set(formattedSpan.span_id.toString(), { - span: undefined, - formattedSpan, - emitted: false, - }) - } - this.export(trace) - } - - /** - * Applies a late framework error to every completed suite retained by the current finalization boundary. - * - * @param {Error} error - * @returns {void} - */ - setDeferredTestSuiteError (error) { - for (const { span, formattedSpan, emitted } of this.#deferredTestSuiteSpans.values()) { - if (emitted) continue - if (span) { - span.setTag(TEST_STATUS, 'fail') - span.setTag('error', error) - } else if (formattedSpan) { - formattedSpan.meta[TEST_STATUS] = 'fail' - spanFormat.addError(formattedSpan, error) - } - } - } - - /** - * Appends a completed suite while preserving the ordinary writer call shape outside finalization. - * - * @param {object} formattedSpan - * @param {{ deadline?: number }} [options] - * @returns {boolean|undefined} - */ - #appendDeferredTestSuiteSpan (formattedSpan, options) { - const appended = options === undefined - ? this._writer.append([formattedSpan]) - : this._writer.append([formattedSpan], options) - - if (appended !== false && this._config.isCiVisibility) { - incrementCountMetric(TELEMETRY_EVENTS_ENQUEUED_FOR_SERIALIZATION) - } - return appended - } - - /** - * Appends completed suites before their module and session parents are exported. - * - * @param {{ deadline?: number }} [options] - * @returns {void} - */ - exportDeferredTestSuiteSpans (options) { - if (!this._writer || !this.canReportSessionTraces()) return - - for (const [spanId, deferredTestSuiteSpan] of this.#deferredTestSuiteSpans) { - const { span, formattedSpan, emitted } = deferredTestSuiteSpan - if (emitted) continue - const updatedSpan = span && spanFormat(span) - if (span && formattedSpan) { - formattedSpan.error = updatedSpan.error - Object.assign(formattedSpan.meta, updatedSpan.meta) - Object.assign(formattedSpan.metrics, updatedSpan.metrics) - if (this.#appendDeferredTestSuiteSpan(formattedSpan, options) === false) continue - this.#deferredTestSuiteSpans.delete(spanId) - } else if (formattedSpan) { - if (this.#appendDeferredTestSuiteSpan(formattedSpan, options) === false) continue - this.#deferredTestSuiteSpans.delete(spanId) - } else { - if (this.#appendDeferredTestSuiteSpan(updatedSpan, options) === false) continue - deferredTestSuiteSpan.emitted = true - } - } - } - - /** - * Discards completed suites that the selected transport cannot deliver. - * - * @returns {void} - */ - resetDeferredTestSuiteSpans () { - this.#deferredTestSuiteSpans.clear() - } - /** * Retries session, module, and suite traces rejected by writer backpressure within the final deadline. * @@ -837,7 +704,7 @@ class CiVisibilityExporter extends BufferingExporter { if (isFinalFlush && !this._isInitialized && this._traceBuffer.length === 0 && this._coverageBuffer.length === 0 && - this.#deferredTestSuiteSpans.size === 0 && this.#deferredTestSessionTraces.length === 0 && + this.#deferredTestSessionTraces.length === 0 && this.#pendingScreenshotUploads.size === 0) { onDone() return @@ -895,7 +762,6 @@ class CiVisibilityExporter extends BufferingExporter { const options = deadline === undefined ? undefined : { deadline } if (isFinalFlush) { - this.exportDeferredTestSuiteSpans(options) this.#exportDeferredTestSessionTraces(options) } diff --git a/packages/dd-trace/src/plugins/ci_plugin.js b/packages/dd-trace/src/plugins/ci_plugin.js index 6abc8367435..87a69389eef 100644 --- a/packages/dd-trace/src/plugins/ci_plugin.js +++ b/packages/dd-trace/src/plugins/ci_plugin.js @@ -728,12 +728,7 @@ module.exports = class CiPlugin extends Plugin { this._bufferWorkerTrace(missingTestSuite, trace) return } - const exporter = this.tracer._exporter - if (exporter.exportTraceWithDeferredTestSuite) { - exporter.exportTraceWithDeferredTestSuite(trace) - } else { - exporter.export(trace) - } + this.tracer._exporter.export(trace) } /** diff --git a/packages/dd-trace/test/ci-visibility/ci-plugin.spec.js b/packages/dd-trace/test/ci-visibility/ci-plugin.spec.js index 2f54fc320f4..495b458f73b 100644 --- a/packages/dd-trace/test/ci-visibility/ci-plugin.spec.js +++ b/packages/dd-trace/test/ci-visibility/ci-plugin.spec.js @@ -340,23 +340,10 @@ describe('CiPlugin', () => { sinon.assert.calledWith(distributionMetric, 'code_coverage.files', {}, 3) }) - it('defers worker suite events when the exporter supports late test suite updates', () => { + it('exports worker traces normally', () => { const plugin = createPlugin('vitest_worker') - const exportTraceWithDeferredTestSuite = sinon.spy() const exportTrace = sinon.spy() const trace = [{ type: 'test_suite_end', meta: {} }] - plugin.tracer._exporter = { export: exportTrace, exportTraceWithDeferredTestSuite } - - plugin._exportWorkerTraceOrBuffer(trace) - - sinon.assert.calledOnceWithExactly(exportTraceWithDeferredTestSuite, trace) - sinon.assert.notCalled(exportTrace) - }) - - it('exports worker traces normally when late test suite updates are unsupported', () => { - const plugin = createPlugin('vitest_worker') - const exportTrace = sinon.spy() - const trace = [{ type: 'test', meta: {} }] plugin.tracer._exporter = { export: exportTrace } plugin._exportWorkerTraceOrBuffer(trace) diff --git a/packages/dd-trace/test/ci-visibility/exporters/agent-proxy/agent-proxy.spec.js b/packages/dd-trace/test/ci-visibility/exporters/agent-proxy/agent-proxy.spec.js index b659cc66867..8d2e3049950 100644 --- a/packages/dd-trace/test/ci-visibility/exporters/agent-proxy/agent-proxy.spec.js +++ b/packages/dd-trace/test/ci-visibility/exporters/agent-proxy/agent-proxy.spec.js @@ -125,7 +125,7 @@ describe('AgentProxyCiVisibilityExporter', () => { } }) - it('exports deferred suite events before buffered module and session events after initialization', async () => { + it('exports suite events before buffered module and session events after initialization', async () => { const controlled = createControlledExporter() const suiteEvent = { type: 'test_suite_end', span_id: '1' } const moduleAndSessionEvents = [ @@ -134,7 +134,7 @@ describe('AgentProxyCiVisibilityExporter', () => { ] const done = sinon.spy() - controlled.exporter.exportTraceWithDeferredTestSuite([suiteEvent]) + controlled.exporter.export([suiteEvent]) controlled.exporter.export(moduleAndSessionEvents) controlled.exporter.flush(done) @@ -146,38 +146,6 @@ describe('AgentProxyCiVisibilityExporter', () => { sinon.assert.calledOnceWithExactly(done, undefined) }) - it('retains deferred suites for reporter errors when initialization finishes before finalization', async () => { - const controlled = createControlledExporter() - const suiteEvent = { - type: 'test_suite_end', - span_id: '1', - error: 0, - meta: { 'test.status': 'pass' }, - metrics: {}, - } - const reporterError = new Error('custom reporter failed') - const moduleAndSessionEvents = [ - { type: 'test_module_end' }, - { type: 'test_session_end' }, - ] - - controlled.exporter.exportTraceWithDeferredTestSuite([suiteEvent]) - controlled.finishAgentInfo(null, { endpoints: ['/evp_proxy/v2'] }) - await Promise.resolve() - - sinon.assert.notCalled(controlled.writers[0].append) - - controlled.exporter.setDeferredTestSuiteError(reporterError) - controlled.exporter.exportDeferredTestSuiteSpans() - controlled.exporter.export(moduleAndSessionEvents) - - sinon.assert.calledWithExactly(controlled.writers[0].append.firstCall, [suiteEvent]) - assert.strictEqual(suiteEvent.meta['test.status'], 'fail') - assert.strictEqual(suiteEvent.error, 1) - assert.strictEqual(suiteEvent.meta['error.message'], reporterError.message) - sinon.assert.calledWithExactly(controlled.writers[0].append.secondCall, moduleAndSessionEvents) - }) - it('aborts initialization and uses the fallback writer for later sessions', async () => { const clock = sinon.useFakeTimers() try { diff --git a/packages/dd-trace/test/ci-visibility/exporters/ci-visibility-exporter.spec.js b/packages/dd-trace/test/ci-visibility/exporters/ci-visibility-exporter.spec.js index 0aa8229582d..d4ae72166a7 100644 --- a/packages/dd-trace/test/ci-visibility/exporters/ci-visibility-exporter.spec.js +++ b/packages/dd-trace/test/ci-visibility/exporters/ci-visibility-exporter.spec.js @@ -20,7 +20,6 @@ const { createEfdRetryPolicy } = require('../../../src/ci-visibility/efd-retry-p const getConfig = require('../../../src/config') const { defaults: { hostname, port } } = require('../../../src/config/defaults') const ciVisibilityLog = require('../../../src/log') -const actualSpanFormat = require('../../../src/span_format') const { uploadCoverageReport: actualUploadCoverageReportRequest } = require('../../../src/ci-visibility/requests/upload-coverage-report') const { uploadTestScreenshot: actualUploadTestScreenshotRequest } = @@ -30,10 +29,6 @@ const sketchesJsPath = require.resolve('../../../../../vendor/dist/@datadog/sket let uploadCoverageReportRequest = actualUploadCoverageReportRequest let uploadTestScreenshotRequest = actualUploadTestScreenshotRequest -let formatSpan = actualSpanFormat -let incrementCountMetric -const formatSpanStub = (...args) => formatSpan(...args) -formatSpanStub.addError = actualSpanFormat.addError const CiVisibilityExporterBase = proxyquire('../../../src/ci-visibility/exporters/ci-visibility-exporter', { '../requests/upload-coverage-report': { uploadCoverageReport (...args) { @@ -45,13 +40,6 @@ const CiVisibilityExporterBase = proxyquire('../../../src/ci-visibility/exporter return uploadTestScreenshotRequest(...args) }, }, - '../telemetry': { - incrementCountMetric (...args) { - return incrementCountMetric?.(...args) - }, - TELEMETRY_EVENTS_ENQUEUED_FOR_SERIALIZATION: 'events_enqueued_for_serialization', - }, - '../../span_format': formatSpanStub, }) // The real tracer Config always carries a `testOptimization` namespace object. @@ -82,8 +70,6 @@ describe('CI Visibility Exporter', () => { nock.cleanAll() uploadCoverageReportRequest = actualUploadCoverageReportRequest uploadTestScreenshotRequest = actualUploadTestScreenshotRequest - formatSpan = actualSpanFormat - incrementCountMetric = sinon.stub() }) afterEach(() => { @@ -981,6 +967,24 @@ describe('CI Visibility Exporter', () => { ) sinon.assert.notCalled(ciVisibilityExporter._writer.append) }) + + for (const sessionEventType of ['test_suite_end', 'test_module_end', 'test_session_end']) { + it(`should export test events from a trace containing ${sessionEventType}`, () => { + const writer = { + append: sinon.spy(), + flush: sinon.spy(), + setUrl: sinon.spy(), + } + const testEvent = { type: 'test' } + const ciVisibilityExporter = new CiVisibilityExporter({ url }) + ciVisibilityExporter._isInitialized = true + ciVisibilityExporter._writer = writer + + ciVisibilityExporter.export([testEvent, { type: sessionEventType }]) + + sinon.assert.calledOnceWithExactly(writer.append, [testEvent]) + }) + } }) context('is initialized and can use CI Vis protocol', () => { it('should export session traces', () => { @@ -1023,7 +1027,7 @@ describe('CI Visibility Exporter', () => { assert.strictEqual(typeof writer.flush.firstCall.args[1].deadline, 'number') }) - it('retains a completed suite until final flush and applies later span tags', () => { + it('exports a suite trace without retaining it', () => { const writer = { append: sinon.spy(), flush: sinon.spy(done => done?.()), @@ -1033,237 +1037,18 @@ describe('CI Visibility Exporter', () => { ciVisibilityExporter._isInitialized = true ciVisibilityExporter._writer = writer ciVisibilityExporter._canUseCiVisProtocol = true - const spanId = { toString: () => 'suite-span-id' } - const testSuiteSpan = { - context: () => ({ _spanId: spanId }), - } const testEvent = { type: 'test' } const suiteEvent = { type: 'test_suite_end', - span_id: spanId, + span_id: 'suite-span-id', error: 0, meta: { 'test.status': 'pass' }, metrics: {}, } - formatSpan = sinon.stub().returns({ - error: 1, - meta: { - 'error.message': 'late reporter error', - 'test.status': 'fail', - }, - metrics: {}, - }) - ciVisibilityExporter.deferTestSuiteSpan(testSuiteSpan) ciVisibilityExporter.export([testEvent, suiteEvent]) - sinon.assert.calledOnceWithExactly(writer.append, [testEvent]) - ciVisibilityExporter.flush() - sinon.assert.calledOnce(writer.append) - - const done = sinon.spy() - ciVisibilityExporter.flush(done) - - sinon.assert.calledTwice(writer.append) - assert.deepStrictEqual(writer.append.secondCall.args[0], [{ - ...suiteEvent, - error: 1, - meta: { - 'error.message': 'late reporter error', - 'test.status': 'fail', - }, - }]) - sinon.assert.calledOnceWithExactly(formatSpan, testSuiteSpan) - sinon.assert.calledOnceWithExactly(done, undefined) - }) - - it('retains a formatted worker suite and applies a later reporter error', () => { - const writer = { - append: sinon.spy(), - flush: sinon.spy(done => done?.()), - setUrl: sinon.spy(), - } - const ciVisibilityExporter = new CiVisibilityExporter({ url, flushInterval: 0 }) - ciVisibilityExporter._isInitialized = true - ciVisibilityExporter._writer = writer - ciVisibilityExporter._canUseCiVisProtocol = true - const spanId = { toString: () => 'suite-span-id' } - const testEvent = { type: 'test' } - const suiteEvent = { - type: 'test_suite_end', - span_id: spanId, - error: 0, - meta: { 'test.status': 'pass' }, - metrics: {}, - } - const error = new Error('late reporter error') - - ciVisibilityExporter.exportTraceWithDeferredTestSuite([testEvent, suiteEvent]) - ciVisibilityExporter.setDeferredTestSuiteError(error) - - sinon.assert.calledOnceWithExactly(writer.append, [testEvent]) - const done = sinon.spy() - ciVisibilityExporter.flush(done) - - sinon.assert.calledTwice(writer.append) - assert.strictEqual(writer.append.secondCall.args[0][0], suiteEvent) - assert.strictEqual(suiteEvent.error, 1) - assert.strictEqual(suiteEvent.meta['test.status'], 'fail') - assert.strictEqual(suiteEvent.meta['error.message'], error.message) - assert.strictEqual(suiteEvent.meta['error.type'], error.name) - assert.strictEqual(suiteEvent.meta['error.stack'], error.stack) - sinon.assert.calledOnceWithExactly(done, undefined) - }) - - it('serializes a completed suite at final flush before SpanProcessor exports it', () => { - const writer = { - append: sinon.spy(), - flush: sinon.spy(done => done?.()), - setUrl: sinon.spy(), - } - const ciVisibilityExporter = new CiVisibilityExporter({ url, flushInterval: 0, isCiVisibility: true }) - ciVisibilityExporter._isInitialized = true - ciVisibilityExporter._writer = writer - ciVisibilityExporter._canUseCiVisProtocol = true - const spanId = { toString: () => 'suite-span-id' } - const testSuiteSpan = { - context: () => ({ _spanId: spanId }), - } - const suiteEvent = { - type: 'test_suite_end', - span_id: spanId, - error: 1, - meta: { - 'error.message': 'late reporter error', - 'test.status': 'fail', - }, - metrics: {}, - } - const moduleEvent = { type: 'test_module_end' } - const sessionEvent = { type: 'test_session_end' } - formatSpan = sinon.stub().returns(suiteEvent) - const firstDone = sinon.spy() - - ciVisibilityExporter.deferTestSuiteSpan(testSuiteSpan) - ciVisibilityExporter.flush(firstDone) - - sinon.assert.calledOnceWithExactly( - writer.append, - [suiteEvent], - sinon.match({ deadline: sinon.match.number }) - ) - sinon.assert.calledOnceWithExactly( - incrementCountMetric, - 'events_enqueued_for_serialization' - ) - sinon.assert.calledOnceWithExactly(formatSpan, testSuiteSpan) - sinon.assert.calledOnceWithExactly(firstDone, undefined) - - ciVisibilityExporter.export([suiteEvent, moduleEvent, sessionEvent]) - const secondDone = sinon.spy() - ciVisibilityExporter.flush(secondDone) - - sinon.assert.calledTwice(writer.append) - sinon.assert.calledWithExactly(writer.append.secondCall, [moduleEvent, sessionEvent]) - sinon.assert.calledOnceWithExactly(formatSpan, testSuiteSpan) - sinon.assert.calledOnceWithExactly(secondDone, undefined) - }) - - it('retains a deferred suite until a bounded final append is accepted', () => { - const writer = { - append: sinon.stub().onFirstCall().returns(false).onSecondCall().returns(true), - flush: sinon.spy(done => done?.()), - setUrl: sinon.spy(), - } - const ciVisibilityExporter = new CiVisibilityExporter({ url, flushInterval: 0, isCiVisibility: true }) - ciVisibilityExporter._isInitialized = true - ciVisibilityExporter._writer = writer - ciVisibilityExporter._canUseCiVisProtocol = true - const spanId = { toString: () => 'suite-span-id' } - const testSuiteSpan = { - context: () => ({ _spanId: spanId }), - } - const suiteEvent = { - type: 'test_suite_end', - span_id: spanId, - error: 0, - meta: { 'test.status': 'pass' }, - metrics: {}, - } - formatSpan = sinon.stub().returns(suiteEvent) - - ciVisibilityExporter.deferTestSuiteSpan(testSuiteSpan) - ciVisibilityExporter.export([suiteEvent]) - ciVisibilityExporter.exportDeferredTestSuiteSpans() - - sinon.assert.calledOnceWithExactly(writer.append, [suiteEvent]) - sinon.assert.notCalled(incrementCountMetric) - const done = sinon.spy() - ciVisibilityExporter.flush(done) - - sinon.assert.calledTwice(writer.append) - assert.strictEqual(writer.append.secondCall.args[0][0], suiteEvent) - assert.strictEqual(typeof writer.append.secondCall.args[1].deadline, 'number') - sinon.assert.calledOnceWithExactly( - incrementCountMetric, - 'events_enqueued_for_serialization' - ) - sinon.assert.calledOnceWithExactly(done, undefined) - - ciVisibilityExporter.exportDeferredTestSuiteSpans() - sinon.assert.calledTwice(writer.append) - }) - - it('retains module and session events until bounded final appends are accepted', () => { - const writer = { - append: sinon.stub() - .onFirstCall().returns(false) - .onSecondCall().returns(true) - .onThirdCall().returns(true), - flush: sinon.spy(done => done?.()), - setUrl: sinon.spy(), - } - const ciVisibilityExporter = new CiVisibilityExporter({ url, flushInterval: 0 }) - ciVisibilityExporter._isInitialized = true - ciVisibilityExporter._writer = writer - ciVisibilityExporter._canUseCiVisProtocol = true - const spanId = { toString: () => 'suite-span-id' } - const testSuiteSpan = { - context: () => ({ _spanId: spanId }), - } - const suiteEvent = { - type: 'test_suite_end', - span_id: spanId, - error: 0, - meta: { 'test.status': 'pass' }, - metrics: {}, - } - const moduleEvent = { type: 'test_module_end' } - const sessionEvent = { type: 'test_session_end' } - const moduleAndSessionEvents = [moduleEvent, sessionEvent] - formatSpan = sinon.stub().returns(suiteEvent) - - ciVisibilityExporter.deferTestSuiteSpan(testSuiteSpan) - ciVisibilityExporter.export([suiteEvent, ...moduleAndSessionEvents]) - - sinon.assert.calledOnceWithExactly(writer.append, moduleAndSessionEvents) - - const done = sinon.spy() - ciVisibilityExporter.flush(done) - - sinon.assert.calledThrice(writer.append) - sinon.assert.calledWithExactly( - writer.append.secondCall, - [suiteEvent], - sinon.match({ deadline: sinon.match.number }) - ) - sinon.assert.calledWithExactly( - writer.append.thirdCall, - moduleAndSessionEvents, - writer.append.secondCall.args[1] - ) - sinon.assert.calledOnceWithExactly(writer.flush, sinon.match.func, writer.append.secondCall.args[1]) - sinon.assert.calledOnceWithExactly(done, undefined) + sinon.assert.calledOnceWithExactly(writer.append, [testEvent, suiteEvent]) }) }) }) @@ -1285,46 +1070,6 @@ describe('CI Visibility Exporter', () => { } }) - it('waits for initialization when a completed suite is deferred', async () => { - const writer = { - append: sinon.spy(), - flush: sinon.spy(done => done()), - setUrl: sinon.spy(), - } - const ciVisibilityExporter = new CiVisibilityExporter({ url }) - const spanId = { toString: () => 'suite-span-id' } - const testSuiteSpan = { - context: () => ({ _spanId: spanId }), - } - const suiteEvent = { - type: 'test_suite_end', - span_id: spanId, - error: 0, - meta: {}, - metrics: {}, - } - formatSpan = sinon.stub().returns(suiteEvent) - const done = sinon.spy() - - ciVisibilityExporter.deferTestSuiteSpan(testSuiteSpan) - ciVisibilityExporter.export([suiteEvent]) - ciVisibilityExporter.flush(done) - sinon.assert.notCalled(done) - - ciVisibilityExporter._writer = writer - ciVisibilityExporter._isInitialized = true - ciVisibilityExporter._resolveCanUseCiVisProtocol(true) - await Promise.resolve() - - sinon.assert.calledOnceWithExactly( - writer.append, - [suiteEvent], - sinon.match({ deadline: sinon.match.number }) - ) - sinon.assert.calledOnce(writer.flush) - sinon.assert.calledOnceWithExactly(done, undefined) - }) - for (const [payloadType, writerProperty, exportPayload] of [ ['trace', '_writer', exporter => exporter.export([{ type: 'test' }])], ['coverage', '_coverageWriter', exporter => exporter.exportCoverage({})], @@ -1371,39 +1116,6 @@ describe('CI Visibility Exporter', () => { sinon.assert.calledTwice(writer.flush) }) - it('starts a new final flush after a suite is deferred', () => { - const writer = { - append: sinon.spy(), - flush: sinon.spy(done => done?.()), - setUrl: sinon.spy(), - } - const ciVisibilityExporter = new CiVisibilityExporter({ url, flushInterval: 0 }) - ciVisibilityExporter._isInitialized = true - ciVisibilityExporter._canUseCiVisProtocol = true - ciVisibilityExporter._writer = writer - const testSuiteSpan = { - context: () => ({ _spanId: { toString: () => 'suite-span-id' } }), - } - const suiteEvent = { - type: 'test_suite_end', - error: 0, - meta: { 'test.status': 'pass' }, - metrics: {}, - } - formatSpan = sinon.stub().returns(suiteEvent) - - const firstDone = sinon.spy() - ciVisibilityExporter.flush(firstDone) - ciVisibilityExporter.deferTestSuiteSpan(testSuiteSpan) - const secondDone = sinon.spy() - ciVisibilityExporter.flush(secondDone) - - sinon.assert.calledTwice(writer.flush) - sinon.assert.calledOnceWithExactly(writer.append, [suiteEvent], sinon.match({ deadline: sinon.match.number })) - sinon.assert.calledOnceWithExactly(firstDone, undefined) - sinon.assert.calledOnceWithExactly(secondDone, undefined) - }) - it('does not coalesce new test data into an active final flush', () => { const flushCallbacks = [] const writer = { From cc2107991574095cd9d2d727295639c68704146d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Antonio=20Fern=C3=A1ndez=20de=20Alba?= Date: Fri, 28 Aug 2026 12:21:08 +0200 Subject: [PATCH 03/99] fix(test-optimization): use a 10-second flush interval (#10018) --- ci/init.js | 2 +- packages/dd-trace/test/ci-visibility/validation-init.spec.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/init.js b/ci/init.js index 75b7f5ee7b3..7855f1adcd5 100644 --- a/ci/init.js +++ b/ci/init.js @@ -7,7 +7,7 @@ const { getEnvironmentVariable, getValueFromEnvSources } = require('../packages/ const { isFalse, isTrue } = require('../packages/dd-trace/src/util') const PACKAGE_MANAGERS = ['npm', 'yarn', 'pnpm'] -const DEFAULT_FLUSH_INTERVAL = 5000 +const DEFAULT_FLUSH_INTERVAL = 10_000 const JEST_FLUSH_INTERVAL = 0 const VITEST_NO_WORKER_INIT_ACTIVE_ENV = 'DD_TEST_OPT_VITEST_NO_WORKER_INIT_ACTIVE' const VALIDATION_MODE_ENV = '_DD_TEST_OPTIMIZATION_VALIDATION_MODE' diff --git a/packages/dd-trace/test/ci-visibility/validation-init.spec.js b/packages/dd-trace/test/ci-visibility/validation-init.spec.js index 3c89c885307..ef163756782 100644 --- a/packages/dd-trace/test/ci-visibility/validation-init.spec.js +++ b/packages/dd-trace/test/ci-visibility/validation-init.spec.js @@ -97,7 +97,7 @@ describe('Test Optimization validation initialization', () => { assert.deepStrictEqual(tracer.init.firstCall.args[0], { startupLogs: false, isCiVisibility: true, - flushInterval: 5000, + flushInterval: 10_000, experimental: { exporter: 'ci_validation' }, }) }) From d9d117cf7675b13ec210683557853a998aff237c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Antonio=20Fern=C3=A1ndez=20de=20Alba?= Date: Fri, 28 Aug 2026 12:21:54 +0200 Subject: [PATCH 04/99] fix(test-optimization): retry transient HTTP responses (#10039) --- .../src/ci-visibility/exporters/request.js | 30 ++++-- .../ci-visibility/exporters/request.spec.js | 102 ++++++++++++++++++ 2 files changed, 121 insertions(+), 11 deletions(-) diff --git a/packages/dd-trace/src/ci-visibility/exporters/request.js b/packages/dd-trace/src/ci-visibility/exporters/request.js index 882ac27f4ea..46b740abfbf 100644 --- a/packages/dd-trace/src/ci-visibility/exporters/request.js +++ b/packages/dd-trace/src/ci-visibility/exporters/request.js @@ -11,6 +11,14 @@ const { } = require('../../exporters/common/retry') const { getRateLimitResetDelay } = require('../requests/rate-limit') +/** + * @param {number} statusCode + * @returns {boolean} + */ +function isRetriableHttpStatusCode (statusCode) { + return statusCode === 408 || statusCode === 429 || (statusCode >= 500 && statusCode <= 599) +} + /** * @param {AbortSignal} signal * @returns {Error} @@ -153,15 +161,10 @@ function requestBuffered (data, options, callback) { } const responseStatus = statusCode ?? error.status - const isRetriableHttpError = options.deadline !== undefined && - (responseStatus === 429 || responseStatus >= 500) - if (options.retry === false || (attemptIndex >= getMaxAttempts(attemptOptions) && - (isRetriableNetworkError(error) || isRetriableHttpError))) { - complete(error, result, statusCode, headers) - return - } - - if (!isRetriableNetworkError(error) && !isRetriableHttpError) { + const isRetriableError = isRetriableNetworkError(error) || isRetriableHttpStatusCode(responseStatus) + const reachedBackgroundAttemptLimit = + options.deadline === undefined && attemptIndex >= getMaxAttempts(attemptOptions) + if (options.retry === false || !isRetriableError || reachedBackgroundAttemptLimit) { complete(error, result, statusCode, headers) return } @@ -170,8 +173,13 @@ function requestBuffered (data, options, callback) { if (responseStatus === 429) { const resetDelay = getRateLimitResetDelay(headers) if (Number.isFinite(resetDelay)) { - const retryRemaining = options.deadline - Date.now() - if (resetDelay > RATE_LIMIT_MAX_WAIT_MS || resetDelay >= retryRemaining) { + if (options.deadline !== undefined) { + const retryRemaining = options.deadline - Date.now() + if (resetDelay >= retryRemaining) { + complete(error, result, statusCode, headers) + return + } + } else if (resetDelay > RATE_LIMIT_MAX_WAIT_MS) { complete(error, result, statusCode, headers) return } diff --git a/packages/dd-trace/test/ci-visibility/exporters/request.spec.js b/packages/dd-trace/test/ci-visibility/exporters/request.spec.js index 723c85d3088..34ea5223263 100644 --- a/packages/dd-trace/test/ci-visibility/exporters/request.spec.js +++ b/packages/dd-trace/test/ci-visibility/exporters/request.spec.js @@ -53,6 +53,67 @@ describe('Test Optimization exporter request', () => { sinon.assert.calledOnceWithExactly(done, null, 'ok', 200, {}) }) + it('keeps retrying transient responses while the finalization deadline has capacity', () => { + const done = sinon.spy() + request('payload', { deadline: Date.now() + 30_000 }, done) + const error = Object.assign(new Error('unavailable'), { status: 503 }) + + pendingRequests[0].callback(error, null, 503, {}) + clock.tick(6000) + pendingRequests[1].callback(error, null, 503, {}) + clock.tick(6000) + pendingRequests[2].callback(error, null, 503, {}) + clock.tick(6000) + pendingRequests[3].callback(null, 'ok', 200, {}) + + assert.strictEqual(pendingRequests.length, 4) + sinon.assert.calledOnceWithExactly(done, null, 'ok', 200, {}) + }) + + for (const statusCode of [408, 429, 500, 599]) { + it(`retries a ${statusCode} response during a background flush`, () => { + const done = sinon.spy() + request('payload', {}, done) + const error = Object.assign(new Error('transient response'), { status: statusCode }) + + pendingRequests[0].callback(error, null, statusCode, {}) + clock.tick(5999) + assert.strictEqual(pendingRequests.length, 1) + clock.tick(1) + assert.strictEqual(pendingRequests.length, 2) + + pendingRequests[1].callback(null, 'ok', 200, {}) + sinon.assert.calledOnceWithExactly(done, null, 'ok', 200, {}) + }) + } + + for (const statusCode of [409, 430, 499, 600]) { + it(`does not retry a ${statusCode} response`, () => { + const done = sinon.spy() + request('payload', {}, done) + const error = Object.assign(new Error('non-retriable response'), { status: statusCode }) + + pendingRequests[0].callback(error, null, statusCode, {}) + + assert.strictEqual(pendingRequests.length, 1) + sinon.assert.calledOnceWithExactly(done, error, null, statusCode, {}) + }) + } + + it('keeps the ordinary attempt cap for background HTTP retries', () => { + const done = sinon.spy() + request('payload', {}, done) + const error = Object.assign(new Error('unavailable'), { status: 503 }) + + pendingRequests[0].callback(error, null, 503, {}) + clock.tick(6000) + pendingRequests[1].callback(error, null, 503, {}) + clock.tick(6000) + + assert.strictEqual(pendingRequests.length, 2) + sinon.assert.calledOnceWithExactly(done, error, null, 503, {}) + }) + it('uses the remaining finalization budget for a late 5xx retry', () => { const done = sinon.spy() request('payload', { deadline: Date.now() + 1000, timeout: 2000 }, done) @@ -109,6 +170,47 @@ describe('Test Optimization exporter request', () => { sinon.assert.calledOnceWithExactly(done, error, null, 429, { 'x-ratelimit-reset': '5' }) }) + it('waits for a rate-limit reset during a background flush', () => { + const done = sinon.spy() + request('payload', {}, done) + + const error = Object.assign(new Error('rate limited'), { status: 429 }) + pendingRequests[0].callback(error, null, 429, { 'x-ratelimit-reset': '5' }) + clock.tick(4999) + assert.strictEqual(pendingRequests.length, 1) + clock.tick(1) + assert.strictEqual(pendingRequests.length, 2) + + pendingRequests[1].callback(null, 'ok', 200, {}) + sinon.assert.calledOnceWithExactly(done, null, 'ok', 200, {}) + }) + + it('does not exceed the rate-limit wait cap during a background flush', () => { + const done = sinon.spy() + request('payload', {}, done) + + const error = Object.assign(new Error('rate limited'), { status: 429 }) + pendingRequests[0].callback(error, null, 429, { 'x-ratelimit-reset': '31' }) + + assert.strictEqual(pendingRequests.length, 1) + sinon.assert.calledOnceWithExactly(done, error, null, 429, { 'x-ratelimit-reset': '31' }) + }) + + it('uses a rate-limit reset over the background cap when the finalization deadline allows it', () => { + const done = sinon.spy() + request('payload', { deadline: Date.now() + 45_000 }, done) + + const error = Object.assign(new Error('rate limited'), { status: 429 }) + pendingRequests[0].callback(error, null, 429, { 'x-ratelimit-reset': '31' }) + clock.tick(30_999) + assert.strictEqual(pendingRequests.length, 1) + clock.tick(1) + assert.strictEqual(pendingRequests.length, 2) + + pendingRequests[1].callback(null, 'ok', 200, {}) + sinon.assert.calledOnceWithExactly(done, null, 'ok', 200, {}) + }) + it('preserves ordinary background retry scheduling without a deadline', () => { request('payload', {}, sinon.spy()) From c13243ef12509e0a26f735ddc553981141167149 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Antonio=20Fern=C3=A1ndez=20de=20Alba?= Date: Fri, 28 Aug 2026 12:22:00 +0200 Subject: [PATCH 05/99] chore(deps): bump test frameworks (#10042) --- .../test/plugins/versions/package.json | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/packages/dd-trace/test/plugins/versions/package.json b/packages/dd-trace/test/plugins/versions/package.json index 57d516cf260..ad895f63caa 100644 --- a/packages/dd-trace/test/plugins/versions/package.json +++ b/packages/dd-trace/test/plugins/versions/package.json @@ -48,13 +48,13 @@ "@grpc/proto-loader": "0.8.1", "@hapi/boom": "10.0.1", "@hapi/hapi": "21.4.10", - "@happy-dom/jest-environment": "20.11.6", + "@happy-dom/jest-environment": "20.11.12", "@hono/node-server": "2.1.1", - "@jest/core": "30.4.2", - "@jest/globals": "30.4.1", - "@jest/reporters": "30.4.1", - "@jest/test-sequencer": "30.4.1", - "@jest/transform": "30.4.1", + "@jest/core": "30.5.0", + "@jest/globals": "30.5.0", + "@jest/reporters": "30.5.0", + "@jest/test-sequencer": "30.5.0", + "@jest/transform": "30.5.0", "@koa/router": "15.7.0", "@langchain/anthropic": "1.5.6", "@langchain/classic": "1.0.43", @@ -93,9 +93,9 @@ "@vitest/coverage-v8": "4.1.11", "@vitest/runner": "4.1.11", "@vscode/sqlite3": "5.1.12-vscode", - "@wdio/cli": "9.31.2", - "@wdio/jasmine-framework": "9.31.2", - "@wdio/local-runner": "9.31.2", + "@wdio/cli": "9.31.4", + "@wdio/jasmine-framework": "9.31.3", + "@wdio/local-runner": "9.31.4", "@wdio/mocha-framework": "9.31.2", "aerospike": "6.7.1", "ai": "7.0.66", @@ -108,7 +108,7 @@ "aws-sdk": "2.1693.0", "axios": "1.19.0", "azure-functions-core-tools": "4.13.0", - "babel-jest": "30.4.1", + "babel-jest": "30.5.0", "bluebird": "3.7.2", "body-parser": "2.3.0", "browser-bunyan": "1.8.0", @@ -121,7 +121,7 @@ "cookie": "2.0.1", "cookie-parser": "1.4.7", "couchbase": "4.7.1", - "cypress": "15.21.0", + "cypress": "15.21.1", "cypress-fail-fast": "8.1.0", "dd-trace-api": "1.0.1", "durable-functions": "3.5.0", @@ -146,16 +146,16 @@ "hono": "4.13.3", "ioredis": "6.0.0", "iovalkey": "0.4.0", - "jest": "30.4.2", - "jest-circus": "30.4.2", - "jest-config": "30.4.2", - "jest-each": "30.4.1", - "jest-environment-jsdom": "30.4.1", - "jest-environment-node": "30.4.1", + "jest": "30.5.0", + "jest-circus": "30.5.0", + "jest-config": "30.5.0", + "jest-each": "30.5.0", + "jest-environment-jsdom": "30.5.0", + "jest-environment-node": "30.5.0", "jest-image-snapshot": "6.5.2", - "jest-jasmine2": "30.4.2", - "jest-runtime": "30.4.2", - "jest-worker": "30.4.1", + "jest-jasmine2": "30.5.0", + "jest-runtime": "30.5.0", + "jest-worker": "30.5.0", "kafkajs": "2.2.4", "knex": "3.3.0", "koa": "3.2.1", From 69461e417a0e682a52c5661420effa59fb8cc1a4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 28 Aug 2026 11:31:30 +0000 Subject: [PATCH 06/99] chore(deps): bump the cloud-and-messaging group across 1 directory with 11 updates (#10043) Bumps the cloud-and-messaging group with 11 updates in the /packages/dd-trace/test/plugins/versions directory: | Package | From | To | | --- | --- | --- | | [@aws-sdk/client-bedrock-runtime](https://github.com/aws/aws-sdk-js-v3/tree/HEAD/clients/client-bedrock-runtime) | `3.1116.0` | `3.1117.0` | | [@aws-sdk/client-dynamodb](https://github.com/aws/aws-sdk-js-v3/tree/HEAD/clients/client-dynamodb) | `3.1116.0` | `3.1117.0` | | [@aws-sdk/client-eventbridge](https://github.com/aws/aws-sdk-js-v3/tree/HEAD/clients/client-eventbridge) | `3.1116.0` | `3.1117.0` | | [@aws-sdk/client-kinesis](https://github.com/aws/aws-sdk-js-v3/tree/HEAD/clients/client-kinesis) | `3.1116.0` | `3.1117.0` | | [@aws-sdk/client-lambda](https://github.com/aws/aws-sdk-js-v3/tree/HEAD/clients/client-lambda) | `3.1116.0` | `3.1117.0` | | [@aws-sdk/client-s3](https://github.com/aws/aws-sdk-js-v3/tree/HEAD/clients/client-s3) | `3.1116.0` | `3.1117.0` | | [@aws-sdk/client-sfn](https://github.com/aws/aws-sdk-js-v3/tree/HEAD/clients/client-sfn) | `3.1116.0` | `3.1117.0` | | [@aws-sdk/client-sns](https://github.com/aws/aws-sdk-js-v3/tree/HEAD/clients/client-sns) | `3.1116.0` | `3.1117.0` | | [@aws-sdk/client-sqs](https://github.com/aws/aws-sdk-js-v3/tree/HEAD/clients/client-sqs) | `3.1116.0` | `3.1117.0` | | [azure-functions-core-tools](https://github.com/Azure/azure-functions-core-tools) | `4.13.0` | `4.14.0` | | [bullmq](https://github.com/taskforcesh/bullmq) | `6.2.0` | `6.2.1` | Updates `@aws-sdk/client-bedrock-runtime` from 3.1116.0 to 3.1117.0 - [Release notes](https://github.com/aws/aws-sdk-js-v3/releases) - [Changelog](https://github.com/aws/aws-sdk-js-v3/blob/main/clients/client-bedrock-runtime/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-js-v3/commits/v3.1117.0/clients/client-bedrock-runtime) Updates `@aws-sdk/client-dynamodb` from 3.1116.0 to 3.1117.0 - [Release notes](https://github.com/aws/aws-sdk-js-v3/releases) - [Changelog](https://github.com/aws/aws-sdk-js-v3/blob/main/clients/client-dynamodb/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-js-v3/commits/v3.1117.0/clients/client-dynamodb) Updates `@aws-sdk/client-eventbridge` from 3.1116.0 to 3.1117.0 - [Release notes](https://github.com/aws/aws-sdk-js-v3/releases) - [Changelog](https://github.com/aws/aws-sdk-js-v3/blob/main/clients/client-eventbridge/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-js-v3/commits/v3.1117.0/clients/client-eventbridge) Updates `@aws-sdk/client-kinesis` from 3.1116.0 to 3.1117.0 - [Release notes](https://github.com/aws/aws-sdk-js-v3/releases) - [Changelog](https://github.com/aws/aws-sdk-js-v3/blob/main/clients/client-kinesis/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-js-v3/commits/v3.1117.0/clients/client-kinesis) Updates `@aws-sdk/client-lambda` from 3.1116.0 to 3.1117.0 - [Release notes](https://github.com/aws/aws-sdk-js-v3/releases) - [Changelog](https://github.com/aws/aws-sdk-js-v3/blob/main/clients/client-lambda/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-js-v3/commits/v3.1117.0/clients/client-lambda) Updates `@aws-sdk/client-s3` from 3.1116.0 to 3.1117.0 - [Release notes](https://github.com/aws/aws-sdk-js-v3/releases) - [Changelog](https://github.com/aws/aws-sdk-js-v3/blob/main/clients/client-s3/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-js-v3/commits/v3.1117.0/clients/client-s3) Updates `@aws-sdk/client-sfn` from 3.1116.0 to 3.1117.0 - [Release notes](https://github.com/aws/aws-sdk-js-v3/releases) - [Changelog](https://github.com/aws/aws-sdk-js-v3/blob/main/clients/client-sfn/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-js-v3/commits/v3.1117.0/clients/client-sfn) Updates `@aws-sdk/client-sns` from 3.1116.0 to 3.1117.0 - [Release notes](https://github.com/aws/aws-sdk-js-v3/releases) - [Changelog](https://github.com/aws/aws-sdk-js-v3/blob/main/clients/client-sns/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-js-v3/commits/v3.1117.0/clients/client-sns) Updates `@aws-sdk/client-sqs` from 3.1116.0 to 3.1117.0 - [Release notes](https://github.com/aws/aws-sdk-js-v3/releases) - [Changelog](https://github.com/aws/aws-sdk-js-v3/blob/main/clients/client-sqs/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-js-v3/commits/v3.1117.0/clients/client-sqs) Updates `azure-functions-core-tools` from 4.13.0 to 4.14.0 - [Release notes](https://github.com/Azure/azure-functions-core-tools/releases) - [Changelog](https://github.com/Azure/azure-functions-core-tools/blob/main/release_notes.md) - [Commits](https://github.com/Azure/azure-functions-core-tools/compare/4.13.0...4.14.0) Updates `bullmq` from 6.2.0 to 6.2.1 - [Release notes](https://github.com/taskforcesh/bullmq/releases) - [Commits](https://github.com/taskforcesh/bullmq/compare/v6.2.0...v6.2.1) --- updated-dependencies: - dependency-name: "@aws-sdk/client-bedrock-runtime" dependency-version: 3.1117.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: cloud-and-messaging - dependency-name: "@aws-sdk/client-dynamodb" dependency-version: 3.1117.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: cloud-and-messaging - dependency-name: "@aws-sdk/client-eventbridge" dependency-version: 3.1117.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: cloud-and-messaging - dependency-name: "@aws-sdk/client-kinesis" dependency-version: 3.1117.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: cloud-and-messaging - dependency-name: "@aws-sdk/client-lambda" dependency-version: 3.1117.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: cloud-and-messaging - dependency-name: "@aws-sdk/client-s3" dependency-version: 3.1117.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: cloud-and-messaging - dependency-name: "@aws-sdk/client-sfn" dependency-version: 3.1117.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: cloud-and-messaging - dependency-name: "@aws-sdk/client-sns" dependency-version: 3.1117.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: cloud-and-messaging - dependency-name: "@aws-sdk/client-sqs" dependency-version: 3.1117.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: cloud-and-messaging - dependency-name: azure-functions-core-tools dependency-version: 4.14.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: cloud-and-messaging - dependency-name: bullmq dependency-version: 6.2.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: cloud-and-messaging ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .../test/plugins/versions/package.json | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/packages/dd-trace/test/plugins/versions/package.json b/packages/dd-trace/test/plugins/versions/package.json index ad895f63caa..48375828510 100644 --- a/packages/dd-trace/test/plugins/versions/package.json +++ b/packages/dd-trace/test/plugins/versions/package.json @@ -15,15 +15,15 @@ "@apollo/subgraph": "2.14.4", "@aws/durable-execution-sdk-js": "2.3.0", "@aws/durable-execution-sdk-js-testing": "1.1.3", - "@aws-sdk/client-bedrock-runtime": "3.1116.0", - "@aws-sdk/client-dynamodb": "3.1116.0", - "@aws-sdk/client-eventbridge": "3.1116.0", - "@aws-sdk/client-kinesis": "3.1116.0", - "@aws-sdk/client-lambda": "3.1116.0", - "@aws-sdk/client-s3": "3.1116.0", - "@aws-sdk/client-sfn": "3.1116.0", - "@aws-sdk/client-sns": "3.1116.0", - "@aws-sdk/client-sqs": "3.1116.0", + "@aws-sdk/client-bedrock-runtime": "3.1117.0", + "@aws-sdk/client-dynamodb": "3.1117.0", + "@aws-sdk/client-eventbridge": "3.1117.0", + "@aws-sdk/client-kinesis": "3.1117.0", + "@aws-sdk/client-lambda": "3.1117.0", + "@aws-sdk/client-s3": "3.1117.0", + "@aws-sdk/client-sfn": "3.1117.0", + "@aws-sdk/client-sns": "3.1117.0", + "@aws-sdk/client-sqs": "3.1117.0", "@aws-sdk/node-http-handler": "3.374.0", "@aws-sdk/smithy-client": "3.374.0", "@azure/cosmos": "4.10.0", @@ -107,13 +107,13 @@ "avsc": "5.7.9", "aws-sdk": "2.1693.0", "axios": "1.19.0", - "azure-functions-core-tools": "4.13.0", + "azure-functions-core-tools": "4.14.0", "babel-jest": "30.5.0", "bluebird": "3.7.2", "body-parser": "2.3.0", "browser-bunyan": "1.8.0", "bson": "7.3.2", - "bullmq": "6.2.0", + "bullmq": "6.2.1", "bunyan": "2.0.5", "cassandra-driver": "4.9.0", "collections": "5.1.13", From 5594d5f500f8c7327faa888d84fd18f7813f8463 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 28 Aug 2026 11:32:03 +0000 Subject: [PATCH 07/99] chore(deps): bump hono (#10036) Bumps the web-frameworks group with 1 update in the /packages/dd-trace/test/plugins/versions directory: [hono](https://github.com/honojs/hono). Updates `hono` from 4.13.3 to 4.13.4 - [Release notes](https://github.com/honojs/hono/releases) - [Commits](https://github.com/honojs/hono/compare/v4.13.3...v4.13.4) --- updated-dependencies: - dependency-name: hono dependency-version: 4.13.4 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: web-frameworks ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- packages/dd-trace/test/plugins/versions/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/dd-trace/test/plugins/versions/package.json b/packages/dd-trace/test/plugins/versions/package.json index 48375828510..3f41b98b17c 100644 --- a/packages/dd-trace/test/plugins/versions/package.json +++ b/packages/dd-trace/test/plugins/versions/package.json @@ -143,7 +143,7 @@ "graphql-yoga": "5.22.0", "handlebars": "4.7.9", "hapi": "18.1.0", - "hono": "4.13.3", + "hono": "4.13.4", "ioredis": "6.0.0", "iovalkey": "0.4.0", "jest": "30.5.0", From a012ec7b0ce02c5cedd3b7cbff47ae04bcae6bdf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 28 Aug 2026 11:32:09 +0000 Subject: [PATCH 08/99] chore(deps): bump the testing-and-build group across 1 directory with 2 updates (#10033) Bumps the testing-and-build group with 2 updates in the /packages/dd-trace/test/plugins/versions directory: [electron](https://github.com/electron/electron) and [tinypool](https://github.com/tinylibs/tinypool). Updates `electron` from 43.4.1 to 44.0.0 - [Release notes](https://github.com/electron/electron/releases) - [Commits](https://github.com/electron/electron/compare/v43.4.1...v44.0.0) Updates `tinypool` from 2.1.0 to 2.1.2 - [Release notes](https://github.com/tinylibs/tinypool/releases) - [Commits](https://github.com/tinylibs/tinypool/compare/v2.1.0...v2.1.2) --- updated-dependencies: - dependency-name: electron dependency-version: 44.0.0 dependency-type: direct:production update-type: version-update:semver-major dependency-group: testing-and-build - dependency-name: tinypool dependency-version: 2.1.2 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: testing-and-build ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- packages/dd-trace/test/plugins/versions/package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/dd-trace/test/plugins/versions/package.json b/packages/dd-trace/test/plugins/versions/package.json index 3f41b98b17c..60abce2c657 100644 --- a/packages/dd-trace/test/plugins/versions/package.json +++ b/packages/dd-trace/test/plugins/versions/package.json @@ -127,7 +127,7 @@ "durable-functions": "3.5.0", "ejs": "6.0.1", "elasticsearch": "16.7.3", - "electron": "43.4.1", + "electron": "44.0.0", "esbuild": "0.28.2", "express": "5.2.1", "express-mongo-sanitize": "2.2.0", @@ -229,7 +229,7 @@ "sqlite3": "6.0.1", "stripe": "22.5.0", "tedious": "20.0.0", - "tinypool": "2.1.0", + "tinypool": "2.1.2", "typescript": "7.0.2", "undici": "8.10.0", "vitest": "4.1.11", From 044163c1e4ff6c1f5234edf18d954a78fb410b63 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 28 Aug 2026 11:33:44 +0000 Subject: [PATCH 09/99] chore(deps): bump the test-versions group across 1 directory with 4 updates (#10032) Bumps the test-versions group with 4 updates in the /packages/dd-trace/test/plugins/versions directory: [@anthropic-ai/claude-agent-sdk](https://github.com/anthropics/claude-agent-sdk-typescript), [@types/node](https://github.com/DefinitelyTyped/DefinitelyTyped/tree/HEAD/types/node), [google-gax](https://github.com/googleapis/google-cloud-node/tree/HEAD/core/packages/gax) and [pnpm](https://github.com/pnpm/pnpm/tree/HEAD/pnpm11/pnpm). Updates `@anthropic-ai/claude-agent-sdk` from 0.3.241 to 0.3.243 - [Release notes](https://github.com/anthropics/claude-agent-sdk-typescript/releases) - [Changelog](https://github.com/anthropics/claude-agent-sdk-typescript/blob/main/CHANGELOG.md) - [Commits](https://github.com/anthropics/claude-agent-sdk-typescript/compare/v0.3.241...v0.3.243) Updates `@types/node` from 26.2.0 to 26.3.0 - [Release notes](https://github.com/DefinitelyTyped/DefinitelyTyped/releases) - [Commits](https://github.com/DefinitelyTyped/DefinitelyTyped/commits/HEAD/types/node) Updates `google-gax` from 6.0.2 to 6.0.3 - [Release notes](https://github.com/googleapis/google-cloud-node/releases) - [Changelog](https://github.com/googleapis/google-cloud-node/blob/main/core/packages/gax/CHANGELOG.md) - [Commits](https://github.com/googleapis/google-cloud-node/commits/google-gax-v6.0.3/core/packages/gax) Updates `pnpm` from 11.22.0 to 11.24.0 - [Release notes](https://github.com/pnpm/pnpm/releases) - [Commits](https://github.com/pnpm/pnpm/commits/v11.24.0/pnpm11/pnpm) --- updated-dependencies: - dependency-name: "@anthropic-ai/claude-agent-sdk" dependency-version: 0.3.243 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: test-versions - dependency-name: "@types/node" dependency-version: 26.3.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: test-versions - dependency-name: google-gax dependency-version: 6.0.3 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: test-versions - dependency-name: pnpm dependency-version: 11.24.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: test-versions ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- packages/dd-trace/test/plugins/versions/package.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/dd-trace/test/plugins/versions/package.json b/packages/dd-trace/test/plugins/versions/package.json index 60abce2c657..148aabec877 100644 --- a/packages/dd-trace/test/plugins/versions/package.json +++ b/packages/dd-trace/test/plugins/versions/package.json @@ -8,7 +8,7 @@ "@ai-sdk/anthropic": "4.0.39", "@ai-sdk/google": "4.0.44", "@ai-sdk/openai": "4.0.42", - "@anthropic-ai/claude-agent-sdk": "0.3.241", + "@anthropic-ai/claude-agent-sdk": "0.3.243", "@anthropic-ai/sdk": "0.117.1", "@apollo/gateway": "2.14.4", "@apollo/server": "5.5.1", @@ -88,7 +88,7 @@ "@redis/client": "6.2.1", "@smithy/core": "3.33.3", "@smithy/smithy-client": "4.15.2", - "@types/node": "26.2.0", + "@types/node": "26.3.0", "@vitest/coverage-istanbul": "4.1.11", "@vitest/coverage-v8": "4.1.11", "@vitest/runner": "4.1.11", @@ -136,7 +136,7 @@ "find-my-way": "9.9.0", "fs": "0.0.1-security", "generic-pool": "3.9.0", - "google-gax": "6.0.2", + "google-gax": "6.0.3", "graphql": "16.14.2", "graphql-tag": "2.12.7", "graphql-tools": "9.0.34", @@ -208,7 +208,7 @@ "pino-pretty": "13.1.3", "playwright": "1.62.1", "playwright-core": "1.62.1", - "pnpm": "11.22.0", + "pnpm": "11.24.0", "prisma": "7.9.1", "promise": "8.3.0", "promise-js": "0.0.7", From 5f1e8d082afd421b31d332082d9ca81592e1485d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Antonio=20Fern=C3=A1ndez=20de=20Alba?= Date: Fri, 28 Aug 2026 13:39:16 +0200 Subject: [PATCH 10/99] fix(test-optimization): extend final flush timeout (#10040) --- integration-tests/jest/jest.core.spec.js | 5 +++-- packages/datadog-instrumentations/src/jest.js | 10 ++++++---- packages/dd-trace/src/ci-visibility/final-flush.js | 2 +- .../exporters/agent-proxy/agent-proxy.spec.js | 11 ++++++----- .../exporters/ci-visibility-exporter.spec.js | 10 +++++++--- 5 files changed, 23 insertions(+), 15 deletions(-) diff --git a/integration-tests/jest/jest.core.spec.js b/integration-tests/jest/jest.core.spec.js index c5e4761d80f..c49dbdf3c00 100644 --- a/integration-tests/jest/jest.core.spec.js +++ b/integration-tests/jest/jest.core.spec.js @@ -53,6 +53,7 @@ const { TEST_COMMAND, } = require('../../packages/dd-trace/src/plugins/util/test') const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env') +const { FINAL_FLUSH_TIMEOUT } = require('../../packages/dd-trace/src/ci-visibility/final-flush') const { ERROR_MESSAGE, ERROR_TYPE, ORIGIN_KEY, COMPONENT } = require('../../packages/dd-trace/src/constants') const { DD_MAJOR } = require('../../version') const { version: ddTraceVersion } = require('../../package.json') @@ -2197,9 +2198,9 @@ describe(`jest@${JEST_VERSION} commonJS`, () => { }) it('bounds the final flush if the server is not available and logs an error', async function () { - this.timeout(20_000) + this.timeout(FINAL_FLUSH_TIMEOUT + 20_000) // Very slow intake - receiver.setWaitingTime(30000) + receiver.setWaitingTime(FINAL_FLUSH_TIMEOUT + 30_000) // Needs to run with the CLI if we want --forceExit to work childProcess = exec( 'node ./node_modules/jest/bin/jest --config config-jest.js --forceExit', diff --git a/packages/datadog-instrumentations/src/jest.js b/packages/datadog-instrumentations/src/jest.js index da31299ec5e..0cc3a88f0ee 100644 --- a/packages/datadog-instrumentations/src/jest.js +++ b/packages/datadog-instrumentations/src/jest.js @@ -18,6 +18,8 @@ const { getEfdRetryCountForDuration, hasEfdRetries, } = require('../../dd-trace/src/ci-visibility/efd-retry-policy') +const { FINAL_FLUSH_FALLBACK_DELAY, FINAL_FLUSH_TIMEOUT } = + require('../../dd-trace/src/ci-visibility/final-flush') const { getCoveredFilesFromCoverage, JEST_WORKER_TRACE_PAYLOAD_CODE, @@ -95,10 +97,10 @@ const itrSkippedSuitesCh = channel('ci:jest:itr:skipped-suites') // https://github.com/jestjs/jest/blob/1d682f21c7a35da4d3ab3a1436a357b980ebd0fa/packages/jest-worker/src/types.ts#L37 const CHILD_MESSAGE_CALL = 1 -// Maximum time we'll wait for the tracer to flush -// The exporter has a 10-second bounded final-flush deadline. Leave enough time -// for its completion callback before Jest's --forceExit fallback takes over. -const FLUSH_TIMEOUT = 12_000 +// Maximum time we'll wait for the tracer to flush. +// Let the exporter enforce its hard deadline and invoke its completion callback +// before Jest's --forceExit fallback takes over. +const FLUSH_TIMEOUT = FINAL_FLUSH_TIMEOUT + FINAL_FLUSH_FALLBACK_DELAY + 5000 const JEST_SESSION_STATE = Symbol.for('dd-trace:jest:session') const JEST_BAIL_REPORTER_PATH = require.resolve('./jest/bail-reporter') const DD_JEST_HANDLE_TEST_EVENT_WRAPPED = Symbol('dd-trace:jest:handle-test-event-wrapped') diff --git a/packages/dd-trace/src/ci-visibility/final-flush.js b/packages/dd-trace/src/ci-visibility/final-flush.js index c1b63347c74..67c47e2d001 100644 --- a/packages/dd-trace/src/ci-visibility/final-flush.js +++ b/packages/dd-trace/src/ci-visibility/final-flush.js @@ -1,6 +1,6 @@ 'use strict' -const FINAL_FLUSH_TIMEOUT = 10_000 +const FINAL_FLUSH_TIMEOUT = 60_000 const FINAL_FLUSH_FALLBACK_DELAY = 100 const FINAL_FLUSH_TIMEOUT_CODE = 'ERR_DD_TEST_OPTIMIZATION_FLUSH_TIMEOUT' diff --git a/packages/dd-trace/test/ci-visibility/exporters/agent-proxy/agent-proxy.spec.js b/packages/dd-trace/test/ci-visibility/exporters/agent-proxy/agent-proxy.spec.js index 8d2e3049950..018bd448b2f 100644 --- a/packages/dd-trace/test/ci-visibility/exporters/agent-proxy/agent-proxy.spec.js +++ b/packages/dd-trace/test/ci-visibility/exporters/agent-proxy/agent-proxy.spec.js @@ -15,6 +15,7 @@ const AgentProxyCiVisibilityExporterBase = require('../../../../src/ci-visibilit const AgentlessWriter = require('../../../../src/ci-visibility/exporters/agentless/writer') const DynamicInstrumentationLogsWriter = require('../../../../src/ci-visibility/exporters/agentless/di-logs-writer') const CoverageWriter = require('../../../../src/ci-visibility/exporters/agentless/coverage-writer') +const { FINAL_FLUSH_TIMEOUT } = require('../../../../src/ci-visibility/final-flush') const AgentWriter = require('../../../../src/exporters/agent/writer') const { clearCache } = require('../../../../src/agent/info') const { defaults: { hostname, port } } = require('../../../../src/config/defaults') @@ -108,7 +109,7 @@ describe('AgentProxyCiVisibilityExporter', () => { const requestOptions = controlled.getRequestOptions() assert.strictEqual(requestOptions.signal.aborted, false) - assert.strictEqual(requestOptions.deadline, Date.now() + 10_000) + assert.strictEqual(requestOptions.deadline, Date.now() + FINAL_FLUSH_TIMEOUT) controlled.finishAgentInfo(null, { endpoints: ['/evp_proxy/v2'] }) await Promise.resolve() @@ -159,7 +160,7 @@ describe('AgentProxyCiVisibilityExporter', () => { controlled.exporter.flush(firstDone) const { signal } = controlled.getRequestOptions() - clock.tick(10_000) + clock.tick(FINAL_FLUSH_TIMEOUT) assert.strictEqual(signal.aborted, true) assert.strictEqual(signal.reason.code, 'ERR_DD_TEST_OPTIMIZATION_FLUSH_TIMEOUT') @@ -199,16 +200,16 @@ describe('AgentProxyCiVisibilityExporter', () => { controlled.exporter.flush(firstDone) const requestOptions = controlled.getRequestOptions() - clock.tick(5_000) + clock.tick(FINAL_FLUSH_TIMEOUT / 2) const secondDone = sinon.spy() const secondTrace = [{ type: 'test', name: 'second session' }] controlled.exporter.export(secondTrace) controlled.exporter.flush(secondDone) - assert.strictEqual(requestOptions.deadline, Date.now() + 10_000) + assert.strictEqual(requestOptions.deadline, Date.now() + FINAL_FLUSH_TIMEOUT) - clock.tick(5_000) + clock.tick(FINAL_FLUSH_TIMEOUT / 2) assert.strictEqual(requestOptions.signal.aborted, false) sinon.assert.calledOnce(firstDone) diff --git a/packages/dd-trace/test/ci-visibility/exporters/ci-visibility-exporter.spec.js b/packages/dd-trace/test/ci-visibility/exporters/ci-visibility-exporter.spec.js index d4ae72166a7..60e8ef23f9d 100644 --- a/packages/dd-trace/test/ci-visibility/exporters/ci-visibility-exporter.spec.js +++ b/packages/dd-trace/test/ci-visibility/exporters/ci-visibility-exporter.spec.js @@ -17,6 +17,10 @@ const { assertObjectContains } = require('../../../../../integration-tests/helpe const { version: tracerVersion } = require('../../../../../package.json') require('../../../../dd-trace/test/setup/core') const { createEfdRetryPolicy } = require('../../../src/ci-visibility/efd-retry-policy') +const { + FINAL_FLUSH_FALLBACK_DELAY, + FINAL_FLUSH_TIMEOUT, +} = require('../../../src/ci-visibility/final-flush') const getConfig = require('../../../src/config') const { defaults: { hostname, port } } = require('../../../src/config/defaults') const ciVisibilityLog = require('../../../src/log') @@ -1184,7 +1188,7 @@ describe('CI Visibility Exporter', () => { ciVisibilityExporter.export([{ type: 'test' }]) ciVisibilityExporter.flush(done) - clock.tick(10_100) + clock.tick(FINAL_FLUSH_TIMEOUT + FINAL_FLUSH_FALLBACK_DELAY) sinon.assert.calledOnce(done) const timeoutError = done.firstCall.args[0] @@ -1882,12 +1886,12 @@ describe('CI Visibility Exporter', () => { exporter.uploadTestScreenshot(screenshotOptions, screenshotCallback) exporter.flush(flushCallback) const requestOptions = uploadTestScreenshotRequest.firstCall.args[0] - assert.strictEqual(requestOptions.deadline, 10_000) + assert.strictEqual(requestOptions.deadline, FINAL_FLUSH_TIMEOUT) assert.strictEqual(requestOptions.signal.aborted, false) sinon.assert.notCalled(exporter._writer.flush) sinon.assert.notCalled(flushCallback) - clock.tick(9_999) + clock.tick(FINAL_FLUSH_TIMEOUT - 1) sinon.assert.notCalled(screenshotCallback) sinon.assert.notCalled(flushCallback) From 77756f22fb683d25e006efc322232b6e4ddca016 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Antonio=20Fern=C3=A1ndez=20de=20Alba?= Date: Fri, 28 Aug 2026 13:39:32 +0200 Subject: [PATCH 11/99] fix(test-optimization): increase payload request concurrency (#10038) --- .../exporters/agentless/coverage-writer.js | 4 +- .../exporters/agentless/di-logs-writer.js | 4 +- .../exporters/agentless/writer.js | 4 +- .../src/ci-visibility/exporters/agents.js | 32 +++++++ .../dd-trace/src/exporters/common/agents.js | 34 ++++++-- packages/dd-trace/src/telemetry/send-data.js | 7 ++ .../agentless/coverage-writer.spec.js | 6 ++ .../agentless/di-logs-writer.spec.js | 20 +++++ .../exporters/agentless/writer.spec.js | 6 ++ .../ci-visibility/exporters/agents.spec.js | 85 +++++++++++++++++++ .../dd-trace/test/telemetry/send-data.spec.js | 49 +++++++++++ 11 files changed, 242 insertions(+), 9 deletions(-) create mode 100644 packages/dd-trace/src/ci-visibility/exporters/agents.js create mode 100644 packages/dd-trace/test/ci-visibility/exporters/agents.spec.js diff --git a/packages/dd-trace/src/ci-visibility/exporters/agentless/coverage-writer.js b/packages/dd-trace/src/ci-visibility/exporters/agentless/coverage-writer.js index 852978a778f..ca64fa55a6f 100644 --- a/packages/dd-trace/src/ci-visibility/exporters/agentless/coverage-writer.js +++ b/packages/dd-trace/src/ci-visibility/exporters/agentless/coverage-writer.js @@ -16,6 +16,7 @@ const { } = require('../../../ci-visibility/telemetry') const { CoverageCIVisibilityEncoder } = require('../../../encode/coverage-ci-visibility') const BaseWriter = require('../../../exporters/common/writer') +const { getAgent } = require('../agents') const request = require('../request') const TestOptimizationRequestTracker = require('./request-tracker') @@ -51,6 +52,7 @@ class Writer extends BaseWriter { }, timeout: 15_000, url: this._url, + agent: getAgent(this._url), deadline: flushOptions?.deadline, } @@ -61,7 +63,7 @@ class Writer extends BaseWriter { } // eslint-disable-next-line eslint-rules/eslint-log-printf-style - log.debug(() => `Request to the intake: ${safeJSONStringify(options)}`) + log.debug(() => `Request to the intake: ${safeJSONStringify({ ...options, agent: undefined })}`) const startRequestTime = Date.now() diff --git a/packages/dd-trace/src/ci-visibility/exporters/agentless/di-logs-writer.js b/packages/dd-trace/src/ci-visibility/exporters/agentless/di-logs-writer.js index 5d0a78e1ffc..8dca7f3f09e 100644 --- a/packages/dd-trace/src/ci-visibility/exporters/agentless/di-logs-writer.js +++ b/packages/dd-trace/src/ci-visibility/exporters/agentless/di-logs-writer.js @@ -6,6 +6,7 @@ const { JSONEncoder } = require('../../encode/json-encoder') const { DEBUGGER_INPUT_V1 } = require('../../../debugger/constants') const BaseWriter = require('../../../exporters/common/writer') +const { getAgent } = require('../agents') const request = require('../request') const TestOptimizationRequestTracker = require('./request-tracker') @@ -46,6 +47,7 @@ class DynamicInstrumentationLogsWriter extends BaseWriter { }, timeout: this.timeout, url: this._url, + agent: getAgent(this._url), deadline: flushOptions?.deadline, } @@ -55,7 +57,7 @@ class DynamicInstrumentationLogsWriter extends BaseWriter { } // eslint-disable-next-line eslint-rules/eslint-log-printf-style - log.debug(() => `Request to the logs intake: ${safeJSONStringify(options)}`) + log.debug(() => `Request to the logs intake: ${safeJSONStringify({ ...options, agent: undefined })}`) this.#requestTracker.send(request, data, options, (err, res) => { if (err) { diff --git a/packages/dd-trace/src/ci-visibility/exporters/agentless/writer.js b/packages/dd-trace/src/ci-visibility/exporters/agentless/writer.js index eb77c5f2467..33cf4818d6f 100644 --- a/packages/dd-trace/src/ci-visibility/exporters/agentless/writer.js +++ b/packages/dd-trace/src/ci-visibility/exporters/agentless/writer.js @@ -16,6 +16,7 @@ const { } = require('../../../ci-visibility/telemetry') const { AgentlessCiVisibilityEncoder } = require('../../../encode/agentless-ci-visibility') const BaseWriter = require('../../../exporters/common/writer') +const { getAgent } = require('../agents') const request = require('../request') const TestOptimizationRequestTracker = require('./request-tracker') @@ -52,6 +53,7 @@ class Writer extends BaseWriter { }, timeout: 15_000, url: this._url, + agent: getAgent(this._url), deadline: flushOptions?.deadline, } @@ -62,7 +64,7 @@ class Writer extends BaseWriter { } // eslint-disable-next-line eslint-rules/eslint-log-printf-style - log.debug(() => `Request to the intake: ${safeJSONStringify(options)}`) + log.debug(() => `Request to the intake: ${safeJSONStringify({ ...options, agent: undefined })}`) const startRequestTime = Date.now() diff --git a/packages/dd-trace/src/ci-visibility/exporters/agents.js b/packages/dd-trace/src/ci-visibility/exporters/agents.js new file mode 100644 index 00000000000..c3906932de1 --- /dev/null +++ b/packages/dd-trace/src/ci-visibility/exporters/agents.js @@ -0,0 +1,32 @@ +'use strict' + +const http = require('http') +const https = require('https') + +const { createAgents } = require('../../exporters/common/agents') + +// Finalization can flush several Test Optimization payload types together. A dedicated pool keeps +// that burst from queuing behind the shared exporters' single socket while preserving a hard cap. +const maxSockets = 16 + +const { httpAgent, httpsAgent } = createAgents(maxSockets) + +/** + * Selects the dedicated Test Optimization payload agent for an intake URL. + * + * @param {string|URL|object} url + * @returns {http.Agent|https.Agent} + */ +function getAgent (url) { + const protocol = url?.protocol + if (protocol === 'https:' || protocol === 'https') return httpsAgent + if (protocol === 'http:' || protocol === 'http') return httpAgent + + try { + return new URL(url).protocol === 'https:' ? httpsAgent : httpAgent + } catch { + return String(url).startsWith('https:') ? httpsAgent : httpAgent + } +} + +module.exports = { getAgent } diff --git a/packages/dd-trace/src/exporters/common/agents.js b/packages/dd-trace/src/exporters/common/agents.js index ebc14a97202..1cea71e4c78 100644 --- a/packages/dd-trace/src/exporters/common/agents.js +++ b/packages/dd-trace/src/exporters/common/agents.js @@ -7,9 +7,15 @@ const { storage } = require('../../../../datadog-core') const legacyStorage = storage('legacy') const keepAlive = true -const maxSockets = 1 -function createAgentClass (BaseAgent) { +/** + * Creates an agent class that suppresses tracing around socket lifecycle operations. + * + * @param {typeof http.Agent|typeof https.Agent} BaseAgent + * @param {number} maxSockets + * @returns {typeof http.Agent|typeof https.Agent} + */ +function createAgentClass (BaseAgent, maxSockets) { class CustomAgent extends BaseAgent { constructor () { super({ keepAlive, maxSockets }) @@ -35,10 +41,26 @@ function createAgentClass (BaseAgent) { return CustomAgent } -const HttpAgent = createAgentClass(http.Agent) -const HttpsAgent = createAgentClass(https.Agent) +/** + * Creates isolated HTTP and HTTPS agents with a bounded connection pool. + * + * @param {number} maxSockets + * @returns {{ httpAgent: http.Agent, httpsAgent: https.Agent }} + */ +function createAgents (maxSockets) { + const HttpAgent = createAgentClass(http.Agent, maxSockets) + const HttpsAgent = createAgentClass(https.Agent, maxSockets) + + return { + httpAgent: new HttpAgent(), + httpsAgent: new HttpsAgent(), + } +} + +const { httpAgent, httpsAgent } = createAgents(1) module.exports = { - httpAgent: new HttpAgent(), - httpsAgent: new HttpsAgent(), + createAgents, + httpAgent, + httpsAgent, } diff --git a/packages/dd-trace/src/telemetry/send-data.js b/packages/dd-trace/src/telemetry/send-data.js index 4cd41783955..32981bde2e7 100644 --- a/packages/dd-trace/src/telemetry/send-data.js +++ b/packages/dd-trace/src/telemetry/send-data.js @@ -69,6 +69,7 @@ const log = require('../log') */ let agentTelemetry = true +let getTestOptimizationAgent /** * @param {import('../config/config-base')} config @@ -145,6 +146,10 @@ function sendData (config, application, host, reqType, payload = {}, cb = () => const isCiVisibilityAgentlessMode = isCiVisibility && testOptimization.DD_CIVISIBILITY_AGENTLESS_ENABLED + if (isCiVisibility && getTestOptimizationAgent === undefined) { + ({ getAgent: getTestOptimizationAgent } = require('../ci-visibility/exporters/agents')) + } + if (isCiVisibilityAgentlessMode) { try { url = testOptimization.DD_CIVISIBILITY_AGENTLESS_URL ?? new URL(getAgentlessTelemetryEndpoint(config.site)) @@ -163,6 +168,7 @@ function sendData (config, application, host, reqType, payload = {}, cb = () => path: isCiVisibilityAgentlessMode ? '/api/v2/apmtelemetry' : '/telemetry/proxy/api/v2/apmtelemetry', headers: getHeaders(config, application, reqType), } + if (isCiVisibility) options.agent = getTestOptimizationAgent(url) const data = JSON.stringify({ api_version: 'v2', @@ -197,6 +203,7 @@ function sendData (config, application, host, reqType, payload = {}, cb = () => headers: backendHeader, path: '/api/v2/apmtelemetry', } + if (isCiVisibility) backendOptions.agent = getTestOptimizationAgent(backendUrl) request(data, backendOptions, (error) => { if (error) { log.error('Error sending telemetry data', error) diff --git a/packages/dd-trace/test/ci-visibility/exporters/agentless/coverage-writer.spec.js b/packages/dd-trace/test/ci-visibility/exporters/agentless/coverage-writer.spec.js index f9bdb0ee4c6..d547dbbe5a7 100644 --- a/packages/dd-trace/test/ci-visibility/exporters/agentless/coverage-writer.spec.js +++ b/packages/dd-trace/test/ci-visibility/exporters/agentless/coverage-writer.spec.js @@ -14,6 +14,8 @@ let encoder let url let log let incrementCountMetric +let agent +let getAgent describe('CI Visibility Coverage Writer', () => { beforeEach(() => { @@ -38,12 +40,15 @@ describe('CI Visibility Coverage Writer', () => { error: sinon.spy(), } incrementCountMetric = sinon.stub() + agent = {} + getAgent = sinon.stub().returns(agent) const CoverageCIVisibilityEncoder = function () { return encoder } CoverageWriter = proxyquire('../../../../src/ci-visibility/exporters/agentless/coverage-writer.js', { + '../agents': { getAgent }, '../request': request, '../../../encode/coverage-ci-visibility': { CoverageCIVisibilityEncoder }, '../../../ci-visibility/telemetry': { incrementCountMetric }, @@ -94,6 +99,7 @@ describe('CI Visibility Coverage Writer', () => { url, path: '/api/v2/citestcov', method: 'POST', + agent, }) done() }) diff --git a/packages/dd-trace/test/ci-visibility/exporters/agentless/di-logs-writer.spec.js b/packages/dd-trace/test/ci-visibility/exporters/agentless/di-logs-writer.spec.js index 5eb254bafac..ea07419cfab 100644 --- a/packages/dd-trace/test/ci-visibility/exporters/agentless/di-logs-writer.spec.js +++ b/packages/dd-trace/test/ci-visibility/exporters/agentless/di-logs-writer.spec.js @@ -27,6 +27,26 @@ describe('Test Visibility DI Writer', () => { }) context('agentless', () => { + it('uses the dedicated Test Optimization agent', (done) => { + const agent = {} + const request = sinon.stub().yieldsAsync(null, 'OK', 202) + const TestOptimizationLogsWriter = proxyquire( + '../../../../src/ci-visibility/exporters/agentless/di-logs-writer', + { + '../agents': { getAgent: () => agent }, + '../request': request, + '../../../config': () => ({ DD_API_KEY: '1' }), + } + ) + const logsWriter = new TestOptimizationLogsWriter({ url: 'http://www.example.com' }) + + logsWriter.append({ message: 'test' }) + logsWriter.flush(() => { + sinon.assert.calledWithMatch(request, sinon.match.any, { agent }) + done() + }) + }) + it('can send logs to the logs intake', (done) => { const scope = nock('http://www.example.com') .post('/api/v2/logs', body => { diff --git a/packages/dd-trace/test/ci-visibility/exporters/agentless/writer.spec.js b/packages/dd-trace/test/ci-visibility/exporters/agentless/writer.spec.js index f7b30342fdb..ca74752046a 100644 --- a/packages/dd-trace/test/ci-visibility/exporters/agentless/writer.spec.js +++ b/packages/dd-trace/test/ci-visibility/exporters/agentless/writer.spec.js @@ -15,6 +15,8 @@ let coverageEncoder let url let log let incrementCountMetric +let agent +let getAgent describe('CI Visibility Writer', () => { beforeEach(() => { @@ -37,6 +39,8 @@ describe('CI Visibility Writer', () => { error: sinon.spy(), } incrementCountMetric = sinon.stub() + agent = {} + getAgent = sinon.stub().returns(agent) const AgentlessCiVisibilityEncoder = function () { return encoder @@ -53,6 +57,7 @@ describe('CI Visibility Writer', () => { } Writer = proxyquire('../../../../src/ci-visibility/exporters/agentless/writer', { + '../agents': { getAgent }, '../request': request, '../../../encode/agentless-ci-visibility': { AgentlessCiVisibilityEncoder }, '../../../encode/coverage-ci-visibility': { CoverageCIVisibilityEncoder }, @@ -103,6 +108,7 @@ describe('CI Visibility Writer', () => { headers: { 'Content-Type': 'application/msgpack', }, + agent, }) done() }) diff --git a/packages/dd-trace/test/ci-visibility/exporters/agents.spec.js b/packages/dd-trace/test/ci-visibility/exporters/agents.spec.js new file mode 100644 index 00000000000..78ae3bca58b --- /dev/null +++ b/packages/dd-trace/test/ci-visibility/exporters/agents.spec.js @@ -0,0 +1,85 @@ +'use strict' + +const assert = require('node:assert/strict') +const http = require('http') +const net = require('net') + +const { describe, it } = require('mocha') + +require('../../setup/core') + +const createRequest = http.request + +const { + httpAgent: commonHttpAgent, + httpsAgent: commonHttpsAgent, +} = require('../../../src/exporters/common/agents') +const { getAgent } = require('../../../src/ci-visibility/exporters/agents') + +describe('Test Optimization exporter agents', () => { + const httpAgent = getAgent('http://localhost') + const httpsAgent = getAgent('https://localhost') + + it('keeps the shared exporter agents serialized at one socket', () => { + assert.notStrictEqual(httpAgent, commonHttpAgent) + assert.notStrictEqual(httpsAgent, commonHttpsAgent) + assert.strictEqual(commonHttpAgent.keepAlive, true) + assert.strictEqual(commonHttpAgent.maxSockets, 1) + assert.strictEqual(commonHttpsAgent.keepAlive, true) + assert.strictEqual(commonHttpsAgent.maxSockets, 1) + }) + + it('configures dedicated agents with bounded concurrency and keep-alive', () => { + assert.strictEqual(httpAgent.keepAlive, true) + assert.strictEqual(httpAgent.maxSockets, 16) + assert.strictEqual(httpsAgent.keepAlive, true) + assert.strictEqual(httpsAgent.maxSockets, 16) + }) + + it('selects the agent by protocol for URL objects and strings', () => { + assert.strictEqual(getAgent(new URL('http://localhost')), httpAgent) + assert.strictEqual(getAgent('http://localhost'), httpAgent) + assert.strictEqual(getAgent(new URL('https://localhost')), httpsAgent) + assert.strictEqual(getAgent('https://localhost'), httpsAgent) + }) + + it('returns a stable singleton per protocol', () => { + assert.strictEqual(getAgent('http://a.example'), getAgent('http://b.example')) + assert.strictEqual(getAgent('https://a.example'), getAgent('https://b.example')) + }) + + it('manages the keep-alive socket lifecycle', () => { + const socket = new net.Socket() + const request = {} + + assert.strictEqual(httpAgent.keepSocketAlive(socket), true) + httpAgent.reuseSocket(socket, request) + assert.strictEqual(request.reusedSocket, true) + socket.destroy() + }) + + it('opens sixteen same-origin connections and queues the seventeenth', async () => { + const agent = new httpAgent.constructor() + const lookupCallbacks = [] + const lookup = (hostname, options, callback) => lookupCallbacks.push(callback) + const requests = new Array(17) + + for (let index = 0; index < requests.length; index++) { + const request = createRequest({ agent, hostname: 'test.local', lookup }) + request.on('error', () => {}) + request.end() + requests[index] = request + } + + await new Promise(resolve => setImmediate(resolve)) + + try { + assert.strictEqual(lookupCallbacks.length, 16) + assert.deepStrictEqual(Object.values(agent.sockets).map(sockets => sockets.length), [16]) + assert.deepStrictEqual(Object.values(agent.requests).map(requests => requests.length), [1]) + } finally { + for (const request of requests) request.destroy() + agent.destroy() + } + }) +}) diff --git a/packages/dd-trace/test/telemetry/send-data.spec.js b/packages/dd-trace/test/telemetry/send-data.spec.js index 5c984fe63a0..5c1d7cb8d3d 100644 --- a/packages/dd-trace/test/telemetry/send-data.spec.js +++ b/packages/dd-trace/test/telemetry/send-data.spec.js @@ -7,6 +7,7 @@ const sinon = require('sinon') const proxyquire = require('proxyquire') const { assertObjectContains } = require('../../../../integration-tests/helpers') +const { getAgent } = require('../../src/ci-visibility/exporters/agents') require('../setup/core') describe('sendData', () => { @@ -51,6 +52,7 @@ describe('sendData', () => { hostname: '', port: '12345', }) + assert.strictEqual(options.agent, undefined) }) it('sends telemetry to the configured socket url', () => { @@ -178,6 +180,7 @@ describe('sendData', () => { }) const { url } = options assert.deepStrictEqual(url, new URL('https://instrumentation-telemetry-intake.datadoghq.eu')) + assert.strictEqual(options.agent, getAgent(url)) }) it('uses DD_CIVISIBILITY_AGENTLESS_URL for telemetry when the agentless intake is overridden', () => { @@ -200,6 +203,52 @@ describe('sendData', () => { const options = request.getCall(0).args[1] const { url } = options assert.deepStrictEqual(url, new URL('https://my-intake.example/')) + assert.strictEqual(options.agent, getAgent(url)) + }) + + it('uses the dedicated Test Optimization agent in agent-proxy mode', () => { + const url = new URL('http://127.0.0.1:8126') + sendDataModule.sendData( + { + isCiVisibility: true, + testOptimization: { DD_CIVISIBILITY_AGENTLESS_ENABLED: false }, + tags: { 'runtime-id': '123' }, + url, + }, + application, + host, + 'req-type' + ) + + sinon.assert.calledOnce(request) + const options = request.firstCall.args[1] + assert.strictEqual(options.path, '/telemetry/proxy/api/v2/apmtelemetry') + assert.strictEqual(options.agent, getAgent(url)) + }) + + it('selects the HTTPS Test Optimization agent when falling back to the agentless backend', () => { + request.onFirstCall().yields(new Error('agent unreachable')) + request.onSecondCall().yields(null) + + const url = new URL('http://127.0.0.1:8126') + sendDataModule.sendData( + { + DD_API_KEY: 'secret-key', + isCiVisibility: true, + site: 'datadoghq.eu', + tags: { 'runtime-id': '123' }, + testOptimization: { DD_CIVISIBILITY_AGENTLESS_ENABLED: false }, + url, + }, + application, + host, + 'req-type' + ) + + assert.strictEqual(request.callCount, 2) + assert.strictEqual(request.firstCall.args[1].agent, getAgent(url)) + const fallbackUrl = new URL('https://instrumentation-telemetry-intake.datadoghq.eu') + assert.strictEqual(request.secondCall.args[1].agent, getAgent(fallbackUrl)) }) it('sends the agentless backend telemetry with a URL object when the agent request fails', () => { From 5ff181c053336b9efaec5bb786efeb1900939ed8 Mon Sep 17 00:00:00 2001 From: Ruben Bridgewater Date: Fri, 28 Aug 2026 20:10:18 +0200 Subject: [PATCH 12/99] feat(dogstatsd): enable client self-telemetry (#9698) This enables DogStatsD self-telemetry for runtime and custom metrics while leaving raw clients telemetry-free. It reports submissions, aggregated contexts, and completed transport outcomes every 10 seconds with standard client tags. Failed telemetry payloads are discarded, so persistent delivery failures cannot increase future work. HTTP saturation now reports a non-transmitting error instead of a successful send, while DogStatsD counts the drop without retrying or falling back to UDP. Other HTTP failures fall back immediately. Only a 404 permanently switches transport attribution to UDP, so transient failures preserve HTTP as the primary transport. Both dd-trace-py and dd-trace-rb leave self-telemetry to their DogStatsD client, so this aligns that. Closes: #8630 --------- Co-authored-by: Omri Fried --- benchmark/sirun/dogstatsd/index.js | 98 +++- benchmark/sirun/dogstatsd/meta.json | 2 +- packages/dd-trace/src/dogstatsd.js | 529 +++++++++++++++--- .../dd-trace/src/exporters/common/request.js | 17 +- .../dd-trace/src/runtime_metrics/client.js | 6 +- packages/dd-trace/test/custom-metrics.spec.js | 13 +- packages/dd-trace/test/dogstatsd.spec.js | 513 +++++++++++++++++ .../test/exporters/common/request.spec.js | 56 +- .../dd-trace/test/runtime_metrics.spec.js | 31 +- 9 files changed, 1127 insertions(+), 138 deletions(-) diff --git a/benchmark/sirun/dogstatsd/index.js b/benchmark/sirun/dogstatsd/index.js index 5f475faa606..c532c57d303 100644 --- a/benchmark/sirun/dogstatsd/index.js +++ b/benchmark/sirun/dogstatsd/index.js @@ -1,66 +1,106 @@ 'use strict' const assert = require('node:assert/strict') -const guard = require('../startup-guard') -const { DogStatsDClient, MetricsAggregationClient } = require('../../../packages/dd-trace/src/dogstatsd') +const guard = require('../startup-guard') -const { VARIANT } = process.env +const { BASELINE_OR_CANDIDATE, VARIANT } = process.env const OPERATIONS = Number(process.env.OPERATIONS) +const WITH_AGGREGATION = VARIANT === 'aggregated' + +// Keep transport I/O outside the measured formatting and aggregation path. +const sockets = [] +const payloads = [] + +function discard () {} + +/** + * @param {Buffer} buffer - Datagram payload + */ +function capture (buffer) { + payloads.push(buffer) +} + +const { + DogStatsDClient, + MetricsAggregationClient, +} = require('../../../packages/dd-trace/src/dogstatsd') -// Every metric the tracer emits (runtime metrics, custom metrics) runs through -// DogStatsDClient._add: build the `stat:value|type` line, splice the global and -// per-metric tags, and append to the 1KB datagram buffer (Buffer.from on each -// overflow). The aggregated variant drives the MetricsAggregationClient tag -// tree that runtime metrics build before flushing. The UDP socket is stubbed so -// nothing leaves the process — the bench measures the in-process formatting and -// buffering only. class BenchClient extends DogStatsDClient { + /** + * @returns {object} Stubbed UDP socket + */ _socket () { - return { send () {}, on () {}, unref () {} } + const socket = { send: capture, on () {}, unref () {} } + + sockets.push(socket) + + return socket } } -const client = new BenchClient({ +/** + * @param {string} host - DogStatsD host + * @param {(error: null, address: string, family: number) => void} callback - Lookup completion + */ +function lookup (host, callback) { + callback(null, host, 4) +} + +const options = { host: '127.0.0.1', port: 8125, tags: ['env:bench', 'service:web-app', 'version:1.2.3'], - lookup: (host, cb) => cb(null, host, 4), -}) + lookup, +} +const client = new BenchClient(options, WITH_AGGREGATION) +const aggregationClient = new MetricsAggregationClient(client) const NAME = 'runtime.node.event_loop.delay.max' const FEW_TAGS = ['lang:javascript', 'lang_version:20.0.0'] const MANY_TAGS = [] -for (let i = 0; i < 12; i++) MANY_TAGS.push(`dim_${i}:value_${i}`) +for (let i = 0; i < 12; i++) { + MANY_TAGS.push(`dim_${i}:value_${i}`) +} function preflight () { - client._add(NAME, 42, 'g', FEW_TAGS) - assert.ok(client._buffer.includes(NAME) && client._buffer.includes('env:bench'), - '_add did not format the metric line with global tags') - client._buffer = '' - client._offset = 0 - client._queue = [] + if (WITH_AGGREGATION) { + aggregationClient.gauge(NAME, 42, FEW_TAGS) + aggregationClient.flush(true) + } else { + client._add(NAME, 42, 'g', FEW_TAGS) + client.flush() + } + + const payload = Buffer.concat(payloads).toString() + assert.ok(payload.includes(NAME) && payload.includes('env:bench'), 'the metric did not reach the transport') + // The older baseline source predates client telemetry, but candidate and local runs must exercise it. + if (WITH_AGGREGATION && BASELINE_OR_CANDIDATE !== 'baseline') { + assert.ok(payload.includes('datadog.dogstatsd.client.metrics:'), 'client telemetry is not enabled') + } + + for (const socket of sockets) socket.send = discard + payloads.length = 0 } preflight() guard.loopStart() -if (VARIANT === 'aggregated') { +if (WITH_AGGREGATION) { // The runtime-metrics path: accumulate into the tag tree, then flush walks the // tree and formats every node through the client. Stubbed socket on flush. - const agg = new MetricsAggregationClient(client) for (let i = 0; i < OPERATIONS; i++) { - agg.count(NAME, 1, FEW_TAGS) - agg.gauge('runtime.node.mem.heap_used', i, FEW_TAGS) - if ((i & 0x3FF) === 0) agg.flush() + aggregationClient.count(NAME, 1, FEW_TAGS) + aggregationClient.gauge('runtime.node.mem.heap_used', i, FEW_TAGS) + if ((i & 0x3FF) === 0) aggregationClient.flush() } - agg.flush() + aggregationClient.flush() } else { const tags = VARIANT === 'no-tags' ? undefined : (VARIANT === 'many-tags' ? MANY_TAGS : FEW_TAGS) const type = VARIANT === 'no-tags' ? 'c' : 'g' for (let i = 0; i < OPERATIONS; i++) { client._add(NAME, i, type, tags) - // Drain the datagram queue without sending so memory stays flat. - if ((i & 0x7FF) === 0) client._queue.length = 0 + // Flush through the stubbed transport so memory stays flat. + if ((i & 0x7FF) === 0) client.flush() } } guard.done() diff --git a/benchmark/sirun/dogstatsd/meta.json b/benchmark/sirun/dogstatsd/meta.json index ae14fe2009e..e6c1f860fcf 100644 --- a/benchmark/sirun/dogstatsd/meta.json +++ b/benchmark/sirun/dogstatsd/meta.json @@ -16,7 +16,7 @@ "env": { "VARIANT": "many-tags", "OPERATIONS": "3000000" } }, "aggregated": { - "env": { "VARIANT": "aggregated", "OPERATIONS": "15000000" } + "env": { "VARIANT": "aggregated", "OPERATIONS": "24000000" } } } } diff --git a/packages/dd-trace/src/dogstatsd.js b/packages/dd-trace/src/dogstatsd.js index 1833d00a2be..a96f4dbac9e 100644 --- a/packages/dd-trace/src/dogstatsd.js +++ b/packages/dd-trace/src/dogstatsd.js @@ -2,7 +2,9 @@ const dgram = require('dgram') const isIP = require('net').isIP +const { performance } = require('node:perf_hooks') +const tracerVersion = require('../../../package.json').version const { storage } = require('../../datadog-core') const request = require('./exporters/common/request') const log = require('./log') @@ -14,42 +16,128 @@ const { createServerlessDeliveryTracker } = require('./serverless') const legacyStorage = storage('legacy') const MAX_BUFFER_SIZE = 1024 // limit from the agent +const TELEMETRY_INTERVAL = 10_000 const TYPE_COUNTER = 'c' const TYPE_GAUGE = 'g' const TYPE_DISTRIBUTION = 'd' const TYPE_HISTOGRAM = 'h' +const TYPE_COUNTER_INDEX = 0 +const TYPE_GAUGE_INDEX = 1 +const TYPE_DISTRIBUTION_INDEX = 2 +const TYPE_HISTOGRAM_INDEX = 3 +const TYPE_LABELS = ['count', 'gauge', 'distribution', 'histogram'] + +/** + * @typedef {'c'|'g'|'d'|'h'} MetricType + */ + +/** + * @typedef {object} DogStatsDBufferState + * @property {string} message + * @property {number} offset + * @property {Buffer[]} queue + */ + +/** + * @typedef {object} DogStatsDClientOptions + * @property {string} host + * @property {number} port + * @property {string[]} tags + * @property {typeof import('node:dns').lookup} lookup + * @property {URL|string} [metricsProxyUrl] + */ + +/** + * @typedef {object} DogStatsDTelemetryState + * @property {number[]} aggregatedContextsByType + * @property {number} bytesDropped + * @property {number} bytesSent + * @property {number[]} metricsByType + * @property {number} nextFlush + * @property {number} packetsDropped + * @property {number} packetsSent + * @property {DogStatsDBufferState} payload + */ + +/** + * @typedef {object} MetricNode + * @property {Map} nodes + * @property {number} submissions + * @property {number|Histogram|null} value + */ + +/** + * @callback CaptureMetric + * @param {MetricNode} node + * @param {string} name + * @param {string[]} tags + * @returns {void} + */ + /** * @import { DogStatsD } from "../../../index.d.ts" * @implements {DogStatsD} */ class DogStatsDClient { + #family + #host + #httpOptions #lookup - #tagsPrefix + #metrics = { message: '', offset: 0, queue: [] } + #port #serverlessDeliveryTracker + #tagsPrefix + #telemetryHttpTagsPrefix + #telemetryUdpTagsPrefix + #udp4 + #udp6 + + /** @type {DogStatsDTelemetryState|undefined} */ + telemetry - constructor (options) { + /** + * @param {DogStatsDClientOptions} options - DogStatsD transport options + * @param {boolean} [telemetryEnabled] - Whether to collect client telemetry + */ + constructor (options, telemetryEnabled = false) { this.#lookup = options.lookup if (options.metricsProxyUrl) { - this._httpOptions = { + this.#httpOptions = { method: 'POST', + retry: false, url: options.metricsProxyUrl.toString(), path: '/dogstatsd/v2/proxy', } } - this._host = options.host - this._family = isIP(this._host) - this._port = options.port - this._tags = options.tags - this.#tagsPrefix = this._tags?.length ? `|#${this._tags.join(',')}` : '' - this._queue = [] + this.#family = isIP(options.host) + this.#host = options.host + this.#port = options.port + this.#tagsPrefix = options.tags?.length ? `|#${options.tags.join(',')}` : '' this.#serverlessDeliveryTracker = createServerlessDeliveryTracker() - this._buffer = '' - this._offset = 0 - this._udp4 = this._socket('udp4') - this._udp6 = this._socket('udp6') + + if (telemetryEnabled) { + this.telemetry = { + aggregatedContextsByType: [0, 0, 0, 0], + bytesDropped: 0, + bytesSent: 0, + metricsByType: [0, 0, 0, 0], + nextFlush: performance.now() + TELEMETRY_INTERVAL, + packetsDropped: 0, + packetsSent: 0, + payload: { message: '', offset: 0, queue: [] }, + } + + const separator = this.#tagsPrefix ? ',' : '|#' + const prefix = `${this.#tagsPrefix}${separator}client:nodejs,client_version:${tracerVersion},client_transport:` + this.#telemetryHttpTagsPrefix = `${prefix}http` + this.#telemetryUdpTagsPrefix = `${prefix}udp` + } + + this.#udp4 = this._socket('udp4') + this.#udp6 = this._socket('udp6') } increment (stat, value, tags) { @@ -72,68 +160,218 @@ class DogStatsDClient { this._add(stat, value, TYPE_HISTOGRAM, tags) } - flush (done) { - const queue = this._enqueue() + /** + * @param {boolean|(() => void)} [forceTelemetry] - Whether to ignore the telemetry interval, or completion callback + * @param {() => void} [done] - Called after serverless deliveries complete + * @returns {void} + */ + flush (forceTelemetry = false, done) { + if (typeof forceTelemetry === 'function') { + done = forceTelemetry + forceTelemetry = false + } - if (queue.length === 0) { - if (this.#serverlessDeliveryTracker) return this.#serverlessDeliveryTracker.waitForIdle(done) - return done?.() + let complete + let track + if (done && !this.#serverlessDeliveryTracker) { + let pending = 1 + complete = () => { + if (--pending === 0) done() + } + track = send => { + pending++ + send(complete) + } } - log.debug('Flushing %s metrics via %s', queue.length, this._httpOptions ? 'HTTP' : 'UDP') + this.#flush(this.#metrics, true, track) + if (this.telemetry) this.#flushTelemetry(forceTelemetry, track) + + if (this.#serverlessDeliveryTracker) return this.#serverlessDeliveryTracker.waitForIdle(done) + complete?.() + } - this._queue = [] + /** + * @param {boolean} force - Whether to ignore the telemetry interval + * @param {((send: (done?: () => void) => void) => void)} [track] - Tracks a non-serverless delivery + * @returns {void} + */ + #flushTelemetry (force, track) { + const telemetry = this.telemetry - const send = complete => { - if (this._httpOptions) this._sendHttp(queue, complete) - else this._sendUdp(queue, complete) + const now = performance.now() + + if (!force && now < telemetry.nextFlush) return + + telemetry.nextFlush = now + TELEMETRY_INTERVAL + + let aggregatedContexts = 0 + let metrics = 0 + for (let index = 0; index < TYPE_LABELS.length; index++) { + aggregatedContexts += telemetry.aggregatedContextsByType[index] + metrics += telemetry.metricsByType[index] + } + + this.#addTelemetry('datadog.dogstatsd.client.metrics', metrics) + this.#addTelemetry('datadog.dogstatsd.client.aggregated_context', aggregatedContexts) + for (let index = 0; index < TYPE_LABELS.length; index++) { + this.#addTelemetry( + 'datadog.dogstatsd.client.metrics_by_type', + telemetry.metricsByType[index], + TYPE_LABELS[index] + ) + this.#addTelemetry( + 'datadog.dogstatsd.client.aggregated_context_by_type', + telemetry.aggregatedContextsByType[index], + TYPE_LABELS[index] + ) + } + this.#addTelemetry('datadog.dogstatsd.client.bytes_sent', telemetry.bytesSent) + this.#addTelemetry('datadog.dogstatsd.client.bytes_dropped', telemetry.bytesDropped) + this.#addTelemetry('datadog.dogstatsd.client.packets_sent', telemetry.packetsSent) + this.#addTelemetry('datadog.dogstatsd.client.packets_dropped', telemetry.packetsDropped) + + telemetry.bytesSent = 0 + telemetry.bytesDropped = 0 + telemetry.packetsSent = 0 + telemetry.packetsDropped = 0 + for (let index = 0; index < TYPE_LABELS.length; index++) { + telemetry.aggregatedContextsByType[index] = 0 + telemetry.metricsByType[index] = 0 } - if (this.#serverlessDeliveryTracker) { - this.#serverlessDeliveryTracker.track(send) - return this.#serverlessDeliveryTracker.waitForIdle(done) + + this.#flush(telemetry.payload, false, track) + } + + /** + * @param {DogStatsDBufferState} state - Payload state to flush + * @param {boolean} recordTelemetry - Whether to record the transport outcome + * @param {((send: (done?: () => void) => void) => void)} [track] - Tracks a non-serverless delivery + * @returns {void} + */ + #flush (state, recordTelemetry, track) { + const queue = this._enqueue(state) + + if (queue.length === 0) return + + log.debug('Flushing %s metrics via %s', queue.length, this.#httpOptions ? 'HTTP' : 'UDP') + + state.queue = [] + + const send = complete => { + if (this.#httpOptions) this._sendHttp(queue, recordTelemetry, complete) + else this._sendUdp(queue, recordTelemetry, complete) } - send(done) + + if (this.#serverlessDeliveryTracker) return this.#serverlessDeliveryTracker.track(send) + if (track) return track(send) + send() + } + + /** + * @param {number} bytes - Number of bytes sent + * @returns {void} + */ + #recordSent (bytes) { + const telemetry = this.telemetry + + telemetry.bytesSent += bytes + telemetry.packetsSent++ } - _sendHttp (queue, done) { + /** + * @param {number} bytes - Number of bytes dropped + * @param {number} [packets] - Number of packets dropped + * @returns {void} + */ + #recordDropped (bytes, packets = 1) { + const telemetry = this.telemetry + + telemetry.bytesDropped += bytes + telemetry.packetsDropped += packets + } + + /** + * Send metrics to the agent via HTTP + * + * @param {Buffer[]} queue - The metrics to send + * @param {boolean} recordTelemetry - Whether to record the transport outcome + * @param {() => void} [done] - Called after delivery completes + * @returns {void} + * @memberof DogStatsDClient + */ + _sendHttp (queue, recordTelemetry, done) { const buffer = Buffer.concat(queue) - request(buffer, this._httpOptions, (err) => { - if (err) { - log.error('DogStatsDClient: HTTP error from agent: %s', err.message, err) - if (err.status === 404) { + request(buffer, this.#httpOptions, (error, _result, _statusCode, _headers, dropped) => { + if (dropped) { + if (recordTelemetry && this.telemetry) { + this.#recordDropped(buffer.length) + } + done?.() + } else if (error) { + log.error('DogStatsDClient: HTTP error from agent: %s', error.message, error) + if (error.status === 404) { // Inside this if-block, we have connectivity to the agent, but // we're not getting a 200 from the proxy endpoint. If it's a 404, // then we know we'll never have the endpoint, so just clear out the // options. Either way, we can give UDP a try. - this._httpOptions = undefined + this.#httpOptions = undefined } - this._sendUdp(queue, done) + this._sendUdp(queue, recordTelemetry, done) } else { + if (recordTelemetry && this.telemetry) this.#recordSent(buffer.length) done?.() } }) } - _sendUdp (queue, done) { + /** + * Send metrics to the agent via UDP + * + * @param {Buffer[]} queue - The metrics to send + * @param {boolean} recordTelemetry - Whether to record the transport outcome + * @param {() => void} [done] - Called after delivery completes + * @returns {void} + * @memberof DogStatsDClient + */ + _sendUdp (queue, recordTelemetry, done) { // dgram resolves the local address via the instrumented dns.lookup when it // binds on first send; the noop store keeps that self-traffic off the trace. legacyStorage.run({ noop: true }, () => { - if (this._family === 0) { - this.#lookup(this._host, (error, address, family) => { + if (this.#family === 0) { + this.#lookup(this.#host, (error, address, family) => { if (error) { + if (recordTelemetry && this.telemetry) { + let bytes = 0 + for (const buffer of queue) { + bytes += buffer.length + } + this.#recordDropped(bytes, queue.length) + } log.error('DogStatsDClient: Host not found', error) return done?.() } - this._sendUdpFromQueue(queue, address, family, done) + this._sendUdpFromQueue(queue, address, family, recordTelemetry, done) }) } else { - this._sendUdpFromQueue(queue, this._host, this._family, done) + this._sendUdpFromQueue(queue, this.#host, this.#family, recordTelemetry, done) } }) } - _sendUdpFromQueue (queue, address, family, done) { - const socket = family === 6 ? this._udp6 : this._udp4 + /** + * Send metrics to the agent via UDP from queue + * + * @param {Buffer[]} queue - The metrics to send + * @param {string} address - The address to send the metrics to + * @param {number} family - The family of the address + * @param {boolean} recordTelemetry - Whether to record the transport outcome + * @param {() => void} [done] - Called after every packet completes + * @returns {void} + * @memberof DogStatsDClient + */ + _sendUdpFromQueue (queue, address, family, recordTelemetry, done) { + const socket = family === 6 ? this.#udp6 : this.#udp4 let pending = queue.length const complete = () => { if (--pending === 0) done?.() @@ -141,8 +379,24 @@ class DogStatsDClient { for (const buffer of queue) { log.debug('Sending to DogStatsD: %s', buffer) + + if (!this.telemetry && !done) { + socket.send(buffer, 0, buffer.length, this.#port, address) + continue + } + try { - socket.send(buffer, 0, buffer.length, this._port, address, complete) + socket.send(buffer, 0, buffer.length, this.#port, address, (error) => { + if (error) { + if (recordTelemetry && this.telemetry) { + this.#recordDropped(buffer.length) + } + log.error('DogStatsDClient: UDP error', error) + } else if (recordTelemetry && this.telemetry) { + this.#recordSent(buffer.length) + } + complete() + }) } catch (error) { log.error('DogStatsDClient: UDP error sending metrics', error) complete() @@ -150,43 +404,81 @@ class DogStatsDClient { } } - _add (stat, value, type, tags) { + /** + * Add a metric to the queue + * + * @param {string} stat - The metric name + * @param {number} value - The metric value + * @param {string} type - The metric type + * @param {string[]} tags - The metric tags + * @param {string} [tagsPrefix] - Serialized global tags + * @param {DogStatsDBufferState} [state] - Payload state receiving the metric + * @returns {void} + * @memberof DogStatsDClient + */ + _add (stat, value, type, tags, tagsPrefix = this.#tagsPrefix, state = this.#metrics) { let message = `${stat}:${value}|${type}` if (tags?.length) { - message += this.#tagsPrefix - ? `${this.#tagsPrefix},${tags.join(',')}` + message += tagsPrefix + ? `${tagsPrefix},${tags.join(',')}` : `|#${tags.join(',')}` } else { - message += this.#tagsPrefix + message += tagsPrefix } if (entityId) { message += `|c:${entityId}` } - this._write(`${message}\n`) + this._write(`${message}\n`, state) } - _write (message) { - const offset = Buffer.byteLength(message) + /** + * @param {string} stat - Telemetry metric name + * @param {number} value - Telemetry metric value + * @param {string} [typeLabel] - Submitted or aggregated metric type + * @returns {void} + */ + #addTelemetry (stat, value, typeLabel) { + const tags = typeLabel === undefined ? undefined : [`metrics_type:${typeLabel}`] + const tagsPrefix = this.#httpOptions ? this.#telemetryHttpTagsPrefix : this.#telemetryUdpTagsPrefix - if (this._offset + offset > MAX_BUFFER_SIZE) { - this._enqueue() - } + this._add(stat, value, TYPE_COUNTER, tags, tagsPrefix, this.telemetry.payload) + } - this._offset += offset - this._buffer += message + /** + * Write a message to the queue + * + * @param {string} message - The message to write + * @param {DogStatsDBufferState} [state] - Payload state receiving the message + * @returns {void} + * @memberof DogStatsDClient + */ + _write (message, state = this.#metrics) { + const offset = Buffer.byteLength(message) + if (state.offset + offset > MAX_BUFFER_SIZE) { + this._enqueue(state) + } + state.offset += offset + state.message += message } - _enqueue () { - if (this._offset > 0) { - this._queue.push(Buffer.from(this._buffer)) - this._buffer = '' - this._offset = 0 + /** + * Enqueue a message to the queue + * + * @param {DogStatsDBufferState} [state] - Payload state to enqueue + * @returns {Buffer[]} Queued payloads + * @memberof DogStatsDClient + */ + _enqueue (state = this.#metrics) { + if (state.offset > 0) { + state.queue.push(Buffer.from(state.message)) + state.message = '' + state.offset = 0 } - return this._queue + return state.queue } _socket (type) { @@ -231,19 +523,57 @@ class DogStatsDClient { } } +/** + * @param {DogStatsDClientOptions} options - DogStatsD transport options + * @returns {MetricsAggregationClient} Aggregating client with shared telemetry + */ +function createMetricsAggregationClient (options) { + const client = new DogStatsDClient(options, true) + + return new MetricsAggregationClient(client) +} + class MetricsAggregationClient { + #metricsByType + + /** @type {DogStatsDTelemetryState|undefined} */ + #telemetry + + /** + * @param {DogStatsDClient} client - DogStatsD transport client + */ constructor (client) { this._client = client + this.#telemetry = client.telemetry + this.#metricsByType = this.#telemetry?.metricsByType ?? [0, 0, 0, 0] this.reset() } - flush (done) { - this._captureCounters() - this._captureGauges() - this._captureHistograms() + /** + * @param {boolean|(() => void)} [forceTelemetry] - Whether to ignore the telemetry interval, or completion callback + * @param {() => void} [done] - Called after serverless deliveries complete + * @returns {void} + */ + flush (forceTelemetry = false, done) { + if (typeof forceTelemetry === 'function') { + done = forceTelemetry + forceTelemetry = false + } + + const counters = this._captureCounters() + const gauges = this._captureGauges() + const histograms = this._captureHistograms() + const telemetry = this.#telemetry - this._client.flush(done) + if (telemetry) { + telemetry.aggregatedContextsByType[TYPE_COUNTER_INDEX] += counters + telemetry.aggregatedContextsByType[TYPE_GAUGE_INDEX] += gauges + telemetry.aggregatedContextsByType[TYPE_HISTOGRAM_INDEX] += histograms + } + + if (forceTelemetry) this._client.flush(true, done) + else this._client.flush(done) } reset () { @@ -255,6 +585,7 @@ class MetricsAggregationClient { // TODO: Aggregate with a histogram and send the buckets to the client. distribution (name, value, tags) { this._client.distribution(name, value, tags) + this.#metricsByType[TYPE_DISTRIBUTION_INDEX]++ } boolean (name, value, tags) { @@ -297,24 +628,45 @@ class MetricsAggregationClient { this.count(name, -count, tags) } + /** + * @returns {number} Number of gauge contexts flushed + */ _captureGauges () { - this._captureTree(this._gauges, (node, name, tags) => { + let metrics = 0 + const contexts = this._captureTree(this._gauges, (node, name, tags) => { + metrics += node.submissions this._client.gauge(name, node.value, tags) }) + this.#metricsByType[TYPE_GAUGE_INDEX] += metrics this._gauges.clear() + + return contexts } + /** + * @returns {number} Number of counter contexts flushed + */ _captureCounters () { - this._captureTree(this._counters, (node, name, tags) => { + let metrics = 0 + const contexts = this._captureTree(this._counters, (node, name, tags) => { + metrics += node.submissions this._client.increment(name, node.value, tags) }) + this.#metricsByType[TYPE_COUNTER_INDEX] += metrics this._counters.clear() + + return contexts } + /** + * @returns {number} Number of histogram contexts flushed + */ _captureHistograms () { - this._captureTree(this._histograms, (node, name, tags) => { + let metrics = 0 + const contexts = this._captureTree(this._histograms, (node, name, tags) => { + metrics += node.submissions const stats = node.value this._client.gauge(`${name}.min`, stats.min, tags) @@ -327,25 +679,49 @@ class MetricsAggregationClient { this._client.gauge(`${name}.95percentile`, stats.p95, tags) }) + this.#metricsByType[TYPE_HISTOGRAM_INDEX] += metrics this._histograms.clear() + + return contexts } + /** + * @param {Map} tree - Metric context tree + * @param {CaptureMetric} fn - Called for every context with submissions + * @returns {number} Number of contexts with submissions + */ _captureTree (tree, fn) { + let contexts = 0 + for (const [name, root] of tree) { - this._captureNode(root, name, [], fn) + contexts += this._captureNode(root, name, [], fn) } + + return contexts } + /** + * @param {MetricNode} node - Current metric context node + * @param {string} name - Metric name + * @param {string[]} tags - Current metric tags + * @param {CaptureMetric} fn - Called for every context with submissions + * @returns {number} Number of contexts with submissions + */ _captureNode (node, name, tags, fn) { - if (node.touched) { + let contexts = 0 + + if (node.submissions !== 0) { fn(node, name, tags) + contexts++ } for (const [tag, next] of node.nodes) { tags.push(tag) - this._captureNode(next, name, tags, fn) + contexts += this._captureNode(next, name, tags, fn) tags.pop() } + + return contexts } _ensureTree (tree, name, tags = [], value) { @@ -359,7 +735,7 @@ class MetricsAggregationClient { node = this._ensureNode(node.nodes, tag, value) } - node.touched = true + node.submissions++ return node } @@ -368,7 +744,7 @@ class MetricsAggregationClient { let node = container.get(key) if (!node) { - node = { nodes: new Map(), touched: false, value } + node = { nodes: new Map(), submissions: 0, value } if (typeof key === 'string') { container.set(key, node) @@ -388,14 +764,14 @@ class CustomMetrics { #client constructor (config) { const clientConfig = DogStatsDClient.generateClientConfig(config) - this.#client = new MetricsAggregationClient(new DogStatsDClient(clientConfig)) + this.#client = createMetricsAggregationClient(clientConfig) const flush = this.flush.bind(this) // TODO(bengl) this magic number should be configurable setInterval(flush, 10 * 1000).unref?.() - globalThis[Symbol.for('dd-trace')].beforeExitHandlers.add(flush) + globalThis[Symbol.for('dd-trace')].beforeExitHandlers.add(() => this.#client.flush(true)) registerTelemetryFlusher(done => this.flush(done)) } @@ -420,7 +796,7 @@ class CustomMetrics { } flush (done) { - return this.#client.flush(done) + return this.#client.flush(false, done) } /** @@ -446,4 +822,5 @@ module.exports = { DogStatsDClient, CustomMetrics, MetricsAggregationClient, + createMetricsAggregationClient, } diff --git a/packages/dd-trace/src/exporters/common/request.js b/packages/dd-trace/src/exporters/common/request.js index 77c93deed4f..0c627db1beb 100644 --- a/packages/dd-trace/src/exporters/common/request.js +++ b/packages/dd-trace/src/exporters/common/request.js @@ -30,7 +30,7 @@ let activeBufferSize = 0 * @param {Buffer|string|Readable|Array} data * @param {object} options * @param {(error: Error|null, result?: string|null, statusCode?: number, - * headers?: import('node:http').IncomingHttpHeaders) => void} callback + * headers?: import('node:http').IncomingHttpHeaders, dropped?: boolean) => void} callback */ function request (data, options, callback) { if (!options.headers) { @@ -89,7 +89,8 @@ function request (data, options, callback) { if (!Array.isArray(data)) { dataArray = [data] } - options.headers['Content-Length'] = byteLength(dataArray) + const contentLength = byteLength(dataArray) + options.headers['Content-Length'] = contentLength docker.inject(options.headers) @@ -172,12 +173,14 @@ function request (data, options, callback) { // outside AsyncContextFrame, so a synchronous re-entry would lose the store. /** @param {number} attemptIndex */ const attempt = attemptIndex => { - if (!request.writable) { - log.debug('Maximum number of active requests reached: payload is discarded.') - return callback(null) + if (activeBufferSize + contentLength > maxActiveBufferSize) { + const error = new log.NoTransmitError('Maximum active request buffer size reached: payload is discarded.') + error.code = 'ERR_DD_REQUEST_BUFFER_FULL' + log.debug(error.message) + return callback(error, undefined, undefined, undefined, true) } - activeBufferSize += options.headers['Content-Length'] ?? 0 + activeBufferSize += contentLength legacyStorage.run({ noop: true }, () => { let finished = false @@ -185,7 +188,7 @@ function request (data, options, callback) { const finalize = () => { if (finished) return finished = true - activeBufferSize -= options.headers['Content-Length'] ?? 0 + activeBufferSize -= contentLength } /** diff --git a/packages/dd-trace/src/runtime_metrics/client.js b/packages/dd-trace/src/runtime_metrics/client.js index 1009f15a04d..890fa951657 100644 --- a/packages/dd-trace/src/runtime_metrics/client.js +++ b/packages/dd-trace/src/runtime_metrics/client.js @@ -1,6 +1,6 @@ 'use strict' -const { DogStatsDClient, MetricsAggregationClient } = require('../dogstatsd') +const { DogStatsDClient, createMetricsAggregationClient } = require('../dogstatsd') const processTags = require('../process-tags') /** @@ -13,7 +13,7 @@ const processTags = require('../process-tags') * also tag user-facing custom metrics, inflating their cardinality (and billing). * * @param {import('../config/config-base')} config - Tracer configuration - * @returns {MetricsAggregationClient} + * @returns {ReturnType} */ function createMetricsClient (config) { const clientConfig = DogStatsDClient.generateClientConfig(config) @@ -24,7 +24,7 @@ function createMetricsClient (config) { } } - return new MetricsAggregationClient(new DogStatsDClient(clientConfig)) + return createMetricsAggregationClient(clientConfig) } module.exports = { createMetricsClient } diff --git a/packages/dd-trace/test/custom-metrics.spec.js b/packages/dd-trace/test/custom-metrics.spec.js index 572ebcbccb8..3088bf17a3c 100644 --- a/packages/dd-trace/test/custom-metrics.spec.js +++ b/packages/dd-trace/test/custom-metrics.spec.js @@ -13,11 +13,12 @@ require('./setup/core') describe('Custom Metrics', () => { let httpServer let httpPort - let metricsData + let metricsPayloads let sockets beforeEach((done) => { sockets = [] + metricsPayloads = [] httpServer = http.createServer((req, res) => { let httpData = '' req.on('data', d => { httpData += d.toString() }) @@ -25,7 +26,7 @@ describe('Custom Metrics', () => { res.statusCode = 200 res.end() if (req.url === '/dogstatsd/v2/proxy') { - metricsData = httpData + metricsPayloads.push(httpData) } }) }).listen(0, () => { @@ -56,7 +57,13 @@ describe('Custom Metrics', () => { // eslint-disable-next-line no-console if (stderr) console.error(stderr) - assert.strictEqual(metricsData.split('#')[0], 'page.views.data:1|c|') + assert.strictEqual(metricsPayloads.length, 2) + + const [userMetrics, telemetryMetrics] = metricsPayloads + + assert.strictEqual(userMetrics.split('#')[0], 'page.views.data:1|c|') + assert.match(telemetryMetrics, /datadog\.dogstatsd\.client\.metrics:1\|c\|/) + assert.match(telemetryMetrics, /datadog\.dogstatsd\.client\.metrics_by_type:1\|c\|.*metrics_type:count/) done() }) diff --git a/packages/dd-trace/test/dogstatsd.spec.js b/packages/dd-trace/test/dogstatsd.spec.js index 260c87182a8..36ce73643be 100644 --- a/packages/dd-trace/test/dogstatsd.spec.js +++ b/packages/dd-trace/test/dogstatsd.spec.js @@ -4,6 +4,7 @@ const assert = require('node:assert/strict') const http = require('node:http') const path = require('node:path') const os = require('node:os') +const { performance } = require('node:perf_hooks') const { describe, it, beforeEach, afterEach } = require('mocha') const sinon = require('sinon') @@ -19,6 +20,7 @@ describe('dogstatsd', () => { let DogStatsDClient let CustomMetrics let MetricsAggregationClient + let createMetricsAggregationClient let dgram let udp4 let udp6 @@ -91,6 +93,7 @@ describe('dogstatsd', () => { DogStatsDClient = dogstatsd.DogStatsDClient CustomMetrics = dogstatsd.CustomMetrics MetricsAggregationClient = dogstatsd.MetricsAggregationClient + createMetricsAggregationClient = dogstatsd.createMetricsAggregationClient httpData = [] statusCode = 200 @@ -146,6 +149,20 @@ describe('dogstatsd', () => { }) } + /** + * @param {object} [options] - DogStatsD client overrides + * @returns {MetricsAggregationClient} Client with self-telemetry enabled + */ + function createTelemetryClient (options) { + return createMetricsAggregationClient({ + host: '127.0.0.1', + lookup: dns.lookup, + port: 8125, + tags: [], + ...options, + }) + } + function createCustomMetrics (CustomMetricsCtor = CustomMetrics) { return new CustomMetricsCtor({ dogstatsd: { @@ -157,6 +174,502 @@ describe('dogstatsd', () => { }) } + /** + * @param {Error} [error] - Asynchronous send result + * @returns {Promise[]} Send completions + */ + function stubUdpSend (error) { + const completions = [] + + udp4.send = sinon.stub().callsFake((buffer, offset, length, port, address, callback) => { + completions.push(Promise.resolve().then(() => callback?.(error))) + }) + udp6.send = sinon.stub().callsFake((buffer, offset, length, port, address, callback) => { + completions.push(Promise.resolve().then(() => callback?.(error))) + }) + + return completions + } + + /** + * @param {number} start - First UDP call to include + * @param {number} [end] - First UDP call to exclude + * @returns {string} Concatenated UDP payload + */ + function getUdpPayload (start, end = udp4.send.callCount) { + let payload = '' + + for (let index = start; index < end; index++) { + payload += udp4.send.getCall(index).args[0].toString() + } + + return payload + } + + describe('client telemetry', () => { + it('emits the client and aggregation metrics every 10 seconds with common UDP tags', async () => { + const now = sinon.stub(performance, 'now').returns(0) + const completions = stubUdpSend() + + try { + client = createTelemetryClient() + + client.increment('test.count') + client.increment('test.count') + client.count('test.delta', 1, [], false) + client.gauge('test.gauge', 5) + client.gauge('test.gauge', 6) + client.distribution('test.distribution', 3) + client.histogram('test.histogram', 7) + client.histogram('test.histogram', 8) + client.flush() + + const userPacketCount = udp4.send.callCount + let userBytes = 0 + for (let index = 0; index < userPacketCount; index++) { + userBytes += udp4.send.getCall(index).args[0].length + } + await Promise.all(completions) + + now.returns(9_999) + client.flush() + + sinon.assert.callCount(udp4.send, userPacketCount) + + now.returns(10_000) + client.flush() + + const firstTelemetryEnd = udp4.send.callCount + const telemetry = getUdpPayload(userPacketCount) + + assert.match(telemetry, /client:nodejs/) + assert.match(telemetry, /client_version:\d+\.\d+\.\d+/) + assert.match(telemetry, /client_transport:udp/) + assert.match(telemetry, /datadog\.dogstatsd\.client\.metrics:8\|c\|/) + assert.match(telemetry, /datadog\.dogstatsd\.client\.metrics_by_type:2\|c\|.*metrics_type:count/) + assert.match(telemetry, /datadog\.dogstatsd\.client\.metrics_by_type:3\|c\|.*metrics_type:gauge/) + assert.match(telemetry, /datadog\.dogstatsd\.client\.metrics_by_type:1\|c\|.*metrics_type:distribution/) + assert.match(telemetry, /datadog\.dogstatsd\.client\.metrics_by_type:2\|c\|.*metrics_type:histogram/) + assert.match(telemetry, /datadog\.dogstatsd\.client\.aggregated_context:4\|c\|/) + assert.match( + telemetry, + /datadog\.dogstatsd\.client\.aggregated_context_by_type:1\|c\|.*metrics_type:count/ + ) + assert.match( + telemetry, + /datadog\.dogstatsd\.client\.aggregated_context_by_type:2\|c\|.*metrics_type:gauge/ + ) + assert.match( + telemetry, + /datadog\.dogstatsd\.client\.aggregated_context_by_type:0\|c\|.*metrics_type:distribution/ + ) + assert.match( + telemetry, + /datadog\.dogstatsd\.client\.aggregated_context_by_type:1\|c\|.*metrics_type:histogram/ + ) + assert.match(telemetry, new RegExp(`datadog\\.dogstatsd\\.client\\.bytes_sent:${userBytes}\\|c\\|`)) + assert.match(telemetry, new RegExp(`datadog\\.dogstatsd\\.client\\.packets_sent:${userPacketCount}\\|c\\|`)) + + client.flush() + sinon.assert.callCount(udp4.send, firstTelemetryEnd) + + now.returns(20_000) + client.flush() + + const secondTelemetry = getUdpPayload(firstTelemetryEnd) + assert.match(secondTelemetry, /datadog\.dogstatsd\.client\.metrics:0\|c\|/) + assert.match(secondTelemetry, /datadog\.dogstatsd\.client\.aggregated_context:0\|c\|/) + assert.match(secondTelemetry, /datadog\.dogstatsd\.client\.bytes_sent:0\|c\|/) + assert.match(secondTelemetry, /datadog\.dogstatsd\.client\.packets_sent:0\|c\|/) + } finally { + now.restore() + } + }) + + it('keeps metrics isolated between telemetry clients', async () => { + const now = sinon.stub(performance, 'now').returns(0) + const completions = stubUdpSend() + + try { + const firstClient = createTelemetryClient({ tags: ['client:first'] }) + const secondClient = createTelemetryClient({ tags: ['client:second'] }) + + firstClient.gauge('first.metric', 1) + firstClient.flush() + secondClient.gauge('second.metric', 1) + secondClient.gauge('second.metric', 2) + secondClient.flush() + + const userPacketCount = udp4.send.callCount + await Promise.all(completions) + + now.returns(10_000) + firstClient.flush() + const firstTelemetryEnd = udp4.send.callCount + secondClient.flush() + + const firstTelemetry = getUdpPayload(userPacketCount, firstTelemetryEnd) + const secondTelemetry = getUdpPayload(firstTelemetryEnd) + + assert.doesNotMatch(firstTelemetry, /datadog\.dogstatsd\.client\.metrics:2\|c\|/) + assert.match(firstTelemetry, /datadog\.dogstatsd\.client\.metrics:1\|c\|/) + assert.match(firstTelemetry, /client:first/) + assert.doesNotMatch(firstTelemetry, /client:second/) + assert.doesNotMatch(secondTelemetry, /datadog\.dogstatsd\.client\.metrics:1\|c\|/) + assert.match(secondTelemetry, /datadog\.dogstatsd\.client\.metrics:2\|c\|/) + assert.match(secondTelemetry, /client:second/) + assert.doesNotMatch(secondTelemetry, /client:first/) + + await Promise.all(completions.slice(userPacketCount)) + } finally { + now.restore() + } + }) + + it('keeps the raw transport path callback-free', () => { + const now = sinon.stub(performance, 'now').returns(0) + + try { + client = new MetricsAggregationClient(new DogStatsDClient({ + host: '127.0.0.1', + lookup: dns.lookup, + port: 8125, + tags: [], + })) + client.increment('test.count') + client.flush() + + sinon.assert.calledOnce(udp4.send) + assert.strictEqual(udp4.send.firstCall.args.length, 5) + + now.returns(10_000) + client.flush() + + sinon.assert.calledOnce(udp4.send) + } finally { + now.restore() + } + }) + + it('records asynchronous UDP completions in the next telemetry interval', () => { + const now = sinon.stub(performance, 'now').returns(0) + const callbacks = [] + udp4.send = sinon.stub().callsFake((buffer, offset, length, port, address, callback) => { + if (callback) callbacks.push(callback) + }) + + try { + client = createTelemetryClient() + client.increment('test.count') + client.flush() + + const userPacketCount = udp4.send.callCount + + now.returns(10_000) + client.flush() + + const firstTelemetryEnd = udp4.send.callCount + const firstTelemetry = getUdpPayload(userPacketCount) + assert.match(firstTelemetry, /datadog\.dogstatsd\.client\.bytes_sent:0\|c\|/) + assert.match(firstTelemetry, /datadog\.dogstatsd\.client\.packets_sent:0\|c\|/) + + callbacks[0]() + now.returns(20_000) + client.flush() + + const secondTelemetry = getUdpPayload(firstTelemetryEnd) + assert.match(secondTelemetry, /datadog\.dogstatsd\.client\.bytes_sent:15\|c\|/) + assert.match(secondTelemetry, /datadog\.dogstatsd\.client\.packets_sent:1\|c\|/) + } finally { + now.restore() + } + }) + + it('reports asynchronous UDP send failures', async () => { + const now = sinon.stub(performance, 'now').returns(0) + const completions = stubUdpSend(new Error('send failed')) + + try { + client = createTelemetryClient() + client.gauge('test.avg', 1) + client.flush() + + await Promise.all(completions) + + const userPacketCount = udp4.send.callCount + now.returns(10_000) + client.flush() + + const telemetry = getUdpPayload(userPacketCount) + assert.match(telemetry, /datadog\.dogstatsd\.client\.bytes_dropped:13\|c\|/) + assert.match(telemetry, /datadog\.dogstatsd\.client\.packets_dropped:1\|c\|/) + } finally { + now.restore() + } + }) + + it('reports DNS drops without exposing transport state', () => { + const now = sinon.stub(performance, 'now').returns(0) + const lookup = sinon.stub() + lookup.onFirstCall().callsArgWith(1, new Error('lookup failed')) + lookup.onSecondCall().callsArgWith(1, undefined, '127.0.0.1', 4) + + try { + client = createTelemetryClient({ host: 'invalid', lookup }) + client.gauge('test.avg', 1) + client.flush() + + sinon.assert.notCalled(udp4.send) + + now.returns(10_000) + client.flush() + + const telemetry = getUdpPayload(0) + assert.match(telemetry, /datadog\.dogstatsd\.client\.bytes_dropped:13\|c\|/) + assert.match(telemetry, /datadog\.dogstatsd\.client\.packets_dropped:1\|c\|/) + } finally { + now.restore() + } + }) + + it('drops failed telemetry after DNS recovers', async () => { + const now = sinon.stub(performance, 'now').returns(0) + const completions = stubUdpSend() + const lookup = sinon.stub() + lookup.onFirstCall().callsArgWith(1, undefined, '127.0.0.1', 4) + lookup.onSecondCall().callsArgWith(1, new Error('lookup failed')) + lookup.onThirdCall().callsArgWith(1, new Error('lookup failed')) + lookup.onCall(3).callsArgWith(1, new Error('lookup failed')) + lookup.onCall(4).callsArgWith(1, undefined, '127.0.0.1', 4) + + try { + client = createTelemetryClient({ host: 'dogstatsd.test', lookup }) + client.gauge('test.avg', 1) + client.flush() + await Promise.all(completions) + + const userPacketCount = udp4.send.callCount + now.returns(10_000) + client.flush() + sinon.assert.callCount(udp4.send, userPacketCount) + + now.returns(20_000) + client.flush() + sinon.assert.callCount(udp4.send, userPacketCount) + + now.returns(30_000) + client.flush() + sinon.assert.callCount(udp4.send, userPacketCount) + + now.returns(40_000) + client.flush() + await Promise.all(completions.slice(userPacketCount)) + + const telemetry = getUdpPayload(userPacketCount) + assert.match(telemetry, /datadog\.dogstatsd\.client\.metrics:0\|c\|/) + assert.doesNotMatch(telemetry, /datadog\.dogstatsd\.client\.metrics:1\|c\|/) + assert.strictEqual(telemetry.match(/datadog\.dogstatsd\.client\.metrics:/g)?.length, 1) + } finally { + now.restore() + } + }) + + it('drops failed telemetry datagrams', async () => { + const now = sinon.stub(performance, 'now').returns(0) + const completions = [] + let failedBuffer + + udp4.send = sinon.stub().callsFake((buffer, offset, length, port, address, callback) => { + let error + if (failedBuffer === undefined && buffer.includes('datadog.dogstatsd.client.')) { + failedBuffer = buffer + error = new Error('send failed') + } + completions.push(Promise.resolve().then(() => callback?.(error))) + }) + + try { + client = createTelemetryClient() + client.gauge('test.avg', 1) + client.flush() + await Promise.all(completions) + + const userPacketCount = udp4.send.callCount + now.returns(10_000) + client.flush() + await Promise.all(completions.slice(userPacketCount)) + + const firstTelemetryEnd = udp4.send.callCount + assert.ok(failedBuffer) + + now.returns(20_000) + client.flush() + await Promise.all(completions.slice(firstTelemetryEnd)) + + for (let index = firstTelemetryEnd; index < udp4.send.callCount; index++) { + assert.notStrictEqual(udp4.send.getCall(index).args[0], failedBuffer) + } + const telemetry = getUdpPayload(firstTelemetryEnd) + assert.match(telemetry, /datadog\.dogstatsd\.client\.metrics:0\|c\|/) + assert.doesNotMatch(telemetry, /datadog\.dogstatsd\.client\.metrics:1\|c\|/) + } finally { + now.restore() + } + }) + + it('drops saturated HTTP writes without falling back to UDP or retrying telemetry', () => { + const now = sinon.stub(performance, 'now').returns(0) + const sendRequest = sinon.stub() + const capacityError = new Error('request buffer is full') + capacityError.code = 'ERR_DD_REQUEST_BUFFER_FULL' + sendRequest.onFirstCall().callsArgWith(2, capacityError, undefined, undefined, undefined, true) + sendRequest.onSecondCall().callsArgWith(2, capacityError, undefined, undefined, undefined, true) + sendRequest.onThirdCall().callsArgWith(2, null, '', 200, {}) + const dogstatsd = proxyquire.noPreserveCache().noCallThru()('../src/dogstatsd', { + dgram, + '../../datadog-core': datadogCore, + './exporters/common/docker': docker, + './exporters/common/request': sendRequest, + './log': log, + }) + + try { + const done = sinon.spy() + client = dogstatsd.createMetricsAggregationClient({ + host: '127.0.0.1', + lookup: dns.lookup, + metricsProxyUrl: `http://localhost:${httpPort}`, + port: 8125, + tags: [], + }) + client.gauge('test.avg', 1) + client.flush(done) + + sinon.assert.calledOnce(done) + sinon.assert.notCalled(udp4.send) + + now.returns(10_000) + client.flush() + + const telemetry = sendRequest.secondCall.args[0].toString() + assert.match(telemetry, /datadog\.dogstatsd\.client\.bytes_dropped:13\|c\|/) + assert.match(telemetry, /datadog\.dogstatsd\.client\.packets_dropped:1\|c\|/) + + now.returns(20_000) + client.flush() + + const nextTelemetry = sendRequest.thirdCall.args[0].toString() + assert.match(nextTelemetry, /datadog\.dogstatsd\.client\.bytes_dropped:0\|c\|/) + assert.match(nextTelemetry, /datadog\.dogstatsd\.client\.packets_dropped:0\|c\|/) + assert.doesNotMatch(nextTelemetry, /datadog\.dogstatsd\.client\.bytes_dropped:13\|c\|/) + sinon.assert.notCalled(udp4.send) + } finally { + now.restore() + } + }) + + it('uses HTTP transport tags and a separate telemetry payload', async () => { + const now = sinon.stub(performance, 'now').returns(0) + const payloads = [] + const telemetryReceived = new Promise(resolve => { + assertData = () => { + payloads.push(Buffer.concat(httpData).toString()) + httpData.length = 0 + + if (payloads.length === 1) { + now.returns(10_000) + client.flush() + return + } + + resolve() + } + }) + + try { + client = createTelemetryClient({ metricsProxyUrl: `http://localhost:${httpPort}` }) + client.gauge('test.avg', 1) + client.flush() + + await telemetryReceived + + assert.strictEqual(payloads[0], 'test.avg:1|g\n') + assert.match(payloads[1], /datadog\.dogstatsd\.client\.metrics:1\|c\|/) + assert.match(payloads[1], /client_transport:http/) + } finally { + now.restore() + } + }) + + it('switches the telemetry transport tag after a 404 permanently disables HTTP', async () => { + const now = sinon.stub(performance, 'now').returns(0) + const udpSent = new Promise(resolve => { + udp4.send = sinon.stub().callsFake((buffer, offset, length, port, address, callback) => { + callback?.() + resolve() + }) + }) + statusCode = 404 + + try { + client = createTelemetryClient({ metricsProxyUrl: `http://localhost:${httpPort}` }) + client.gauge('test.avg', 1) + client.flush() + + await udpSent + + const userPacketCount = udp4.send.callCount + now.returns(10_000) + client.flush() + + const telemetry = getUdpPayload(userPacketCount) + assert.match(telemetry, /client_transport:udp/) + assert.match(telemetry, /datadog\.dogstatsd\.client\.bytes_sent:13\|c\|/) + } finally { + now.restore() + } + }) + + it('keeps the HTTP transport tag after a transient UDP fallback', async () => { + const now = sinon.stub(performance, 'now').returns(0) + let resolveUdp + const udpSent = new Promise(resolve => { + resolveUdp = resolve + }) + const httpAttempted = new Promise(resolve => { + assertData = resolve + }) + udp4.send = sinon.stub().callsFake((buffer, offset, length, port, address, callback) => { + callback?.() + resolveUdp() + }) + statusCode = 500 + + try { + client = createTelemetryClient({ metricsProxyUrl: `http://localhost:${httpPort}` }) + client.gauge('test.avg', 1) + client.flush() + + await Promise.all([httpAttempted, udpSent]) + httpData.length = 0 + + const telemetryReceived = new Promise(resolve => { + assertData = resolve + }) + now.returns(10_000) + client.flush() + + await telemetryReceived + + const telemetry = Buffer.concat(httpData).toString() + assert.match(telemetry, /client_transport:http/) + assert.match(telemetry, /datadog\.dogstatsd\.client\.bytes_sent:13\|c\|/) + } finally { + now.restore() + } + }) + }) + it('should send gauges', () => { client = createDogStatsDClient() diff --git a/packages/dd-trace/test/exporters/common/request.spec.js b/packages/dd-trace/test/exporters/common/request.spec.js index 0c6d5d3b90f..5b256f21db7 100644 --- a/packages/dd-trace/test/exporters/common/request.spec.js +++ b/packages/dd-trace/test/exporters/common/request.spec.js @@ -790,6 +790,49 @@ describe('request', function () { }) }) + it('tracks concurrent payload sizes independently when options are shared', () => { + const requests = [] + + /** + * @returns {EventEmitter} Pending request + */ + function createRequest () { + const pending = new EventEmitter() + pending.setTimeout = sinon.stub() + pending.write = sinon.stub() + pending.end = sinon.stub() + requests.push(pending) + + return pending + } + + const accountingRequest = proxyquire('../../../src/exporters/common/request', { + '../../../../datadog-core': { + storage: () => ({ run: runInNoopContext }), + }, + http: { ...http, request: createRequest }, + './docker': docker, + '../../log': log, + './retry': { + ...require('../../../src/exporters/common/retry'), + ...retryStubs, + }, + }) + const large = Buffer.alloc(63 * 1024 * 1024) + const small = Buffer.alloc(1024 * 1024) + const options = { method: 'POST', headers: {} } + + accountingRequest(large, options, sinon.stub()) + accountingRequest(small, options, sinon.stub()) + requests[0].emit('close') + requests[1].emit('close') + + accountingRequest(large, options, sinon.stub()) + + assert.strictEqual(accountingRequest.writable, true) + requests[2].emit('close') + }) + it('should drop requests when too much data is buffered', (done) => { const bufferSize = 8 * 1024 * 1024 const buffer = Buffer.alloc(bufferSize).fill(69) @@ -820,14 +863,15 @@ describe('request', function () { 'Content-Type': 'application/octet-stream', }, }, - (err, res) => { - if (err) return done(err) - - if (res) { + (error, res, statusCode, headers, dropped) => { + if (error) { + assert.strictEqual(error.code, 'ERR_DD_REQUEST_BUFFER_FULL') + assert.strictEqual(dropped, true) + koCount++ + } else { assert.strictEqual(res, 'OK') + assert.strictEqual(dropped, undefined) okCount++ - } else { - koCount++ } if (okCount + koCount === 10) { diff --git a/packages/dd-trace/test/runtime_metrics.spec.js b/packages/dd-trace/test/runtime_metrics.spec.js index 25317b18285..608f74e6f19 100644 --- a/packages/dd-trace/test/runtime_metrics.spec.js +++ b/packages/dd-trace/test/runtime_metrics.spec.js @@ -13,7 +13,7 @@ const { metrics } = require('@opentelemetry/api') require('./setup/core') const { NODE_MAJOR, NODE_MINOR } = require('../../../version') -const { DogStatsDClient } = require('../src/dogstatsd') +const { DogStatsDClient, MetricsAggregationClient } = require('../src/dogstatsd') // On Node versions that support `monitorEventLoopDelay({ samplePerIteration })` // (available in v24.19.0 and v26.5.0) the runtime metrics module unconditionally skips the @@ -30,6 +30,19 @@ const PeriodicMetricReader = require('../src/opentelemetry/metrics/periodic_metr const OtlpTransformer = require('../src/opentelemetry/metrics/otlp_transformer') const otlpRuntimeMetrics = require('../src/runtime_metrics/otlp_runtime_metrics') +/** + * @param {typeof DogStatsDClient} Client - DogStatsD transport test double + * @returns {typeof import('../src/runtime_metrics/client')} Runtime-metrics client factory with real aggregation + */ +function proxyMetricsClient (Client) { + return proxyquire('../src/runtime_metrics/client', { + '../dogstatsd': { + DogStatsDClient: Client, + createMetricsAggregationClient: options => new MetricsAggregationClient(new Client(options)), + }, + }) +} + function createGarbage (count = 50) { let last = {} const obj = last @@ -206,9 +219,7 @@ NATIVE_METRICS_VARIANTS.forEach((nativeMetrics) => { const proxiedObject = { // Exercise the real client factory (incl. process tags) but with the spy DogStatsD client. - './client': proxyquire('../src/runtime_metrics/client', { - '../dogstatsd': { DogStatsDClient: Client }, - }), + './client': proxyMetricsClient(Client), } if (!nativeMetrics) { proxiedObject['@datadog/native-metrics'] = { @@ -551,9 +562,7 @@ NATIVE_METRICS_VARIANTS.forEach((nativeMetrics) => { LocalClient.generateClientConfig = DogStatsDClient.generateClientConfig const localRuntimeMetrics = proxyquire('../src/runtime_metrics/runtime_metrics', { - './client': proxyquire('../src/runtime_metrics/client', { - '../dogstatsd': { DogStatsDClient: LocalClient }, - }), + './client': proxyMetricsClient(LocalClient), '@datadog/native-metrics': nativeMetricsModule, }) @@ -597,9 +606,7 @@ NATIVE_METRICS_VARIANTS.forEach((nativeMetrics) => { LocalClient.generateClientConfig = DogStatsDClient.generateClientConfig const localRuntimeMetrics = proxyquire('../src/runtime_metrics/runtime_metrics', { - './client': proxyquire('../src/runtime_metrics/client', { - '../dogstatsd': { DogStatsDClient: LocalClient }, - }), + './client': proxyMetricsClient(LocalClient), '../../../../version': { NODE_MAJOR: 24, NODE_MINOR: 19 }, '@datadog/native-metrics': { start () { @@ -1007,9 +1014,7 @@ describeSamplePerIteration('runtimeMetrics event loop delay via samplePerIterati }) localRuntimeMetrics = proxyquire('../src/runtime_metrics/runtime_metrics', { - './client': proxyquire('../src/runtime_metrics/client', { - '../dogstatsd': { DogStatsDClient: LocalClient }, - }), + './client': proxyMetricsClient(LocalClient), '@datadog/native-metrics': { start: nativeMetricsStart, stop () {} }, }) From cc3a97f02b9049dc4531531d522a650dab7601d4 Mon Sep 17 00:00:00 2001 From: Ruben Bridgewater Date: Fri, 28 Aug 2026 21:38:25 +0200 Subject: [PATCH 13/99] fix(test-optimization): cap screenshot upload retries (#10056) A persistent media 5xx bypassed the ordinary retry cap because the upload had a finalization deadline, so the Cypress runner waited until the integration process timed out. --- .../src/ci-visibility/exporters/request.js | 6 +++--- .../requests/upload-test-screenshot.js | 1 + .../test/ci-visibility/exporters/request.spec.js | 14 ++++++++++++++ .../requests/upload-test-screenshot.spec.js | 5 +++-- 4 files changed, 21 insertions(+), 5 deletions(-) diff --git a/packages/dd-trace/src/ci-visibility/exporters/request.js b/packages/dd-trace/src/ci-visibility/exporters/request.js index 46b740abfbf..1862106c504 100644 --- a/packages/dd-trace/src/ci-visibility/exporters/request.js +++ b/packages/dd-trace/src/ci-visibility/exporters/request.js @@ -162,9 +162,9 @@ function requestBuffered (data, options, callback) { const responseStatus = statusCode ?? error.status const isRetriableError = isRetriableNetworkError(error) || isRetriableHttpStatusCode(responseStatus) - const reachedBackgroundAttemptLimit = - options.deadline === undefined && attemptIndex >= getMaxAttempts(attemptOptions) - if (options.retry === false || !isRetriableError || reachedBackgroundAttemptLimit) { + const retryUntilDeadline = options.deadline !== undefined && options.retryUntilDeadline !== false + const reachedAttemptLimit = !retryUntilDeadline && attemptIndex >= getMaxAttempts(attemptOptions) + if (options.retry === false || !isRetriableError || reachedAttemptLimit) { complete(error, result, statusCode, headers) return } diff --git a/packages/dd-trace/src/ci-visibility/requests/upload-test-screenshot.js b/packages/dd-trace/src/ci-visibility/requests/upload-test-screenshot.js index 55c230624e3..1c9c0662692 100644 --- a/packages/dd-trace/src/ci-visibility/requests/upload-test-screenshot.js +++ b/packages/dd-trace/src/ci-visibility/requests/upload-test-screenshot.js @@ -130,6 +130,7 @@ function uploadTestScreenshot ( timeout: UPLOAD_TIMEOUT_MS, url, deadline, + retryUntilDeadline: false, signal, } diff --git a/packages/dd-trace/test/ci-visibility/exporters/request.spec.js b/packages/dd-trace/test/ci-visibility/exporters/request.spec.js index 34ea5223263..4edb07bd928 100644 --- a/packages/dd-trace/test/ci-visibility/exporters/request.spec.js +++ b/packages/dd-trace/test/ci-visibility/exporters/request.spec.js @@ -70,6 +70,20 @@ describe('Test Optimization exporter request', () => { sinon.assert.calledOnceWithExactly(done, null, 'ok', 200, {}) }) + it('keeps the ordinary attempt cap when deadline retries are disabled', () => { + const done = sinon.spy() + request('payload', { deadline: Date.now() + 30_000, retryUntilDeadline: false }, done) + const error = Object.assign(new Error('unavailable'), { status: 503 }) + + pendingRequests[0].callback(error, null, 503, {}) + clock.tick(6000) + pendingRequests[1].callback(error, null, 503, {}) + clock.tick(6000) + + assert.strictEqual(pendingRequests.length, 2) + sinon.assert.calledOnceWithExactly(done, error, null, 503, {}) + }) + for (const statusCode of [408, 429, 500, 599]) { it(`retries a ${statusCode} response during a background flush`, () => { const done = sinon.spy() diff --git a/packages/dd-trace/test/ci-visibility/requests/upload-test-screenshot.spec.js b/packages/dd-trace/test/ci-visibility/requests/upload-test-screenshot.spec.js index 46b7906c476..9594080cc90 100644 --- a/packages/dd-trace/test/ci-visibility/requests/upload-test-screenshot.spec.js +++ b/packages/dd-trace/test/ci-visibility/requests/upload-test-screenshot.spec.js @@ -38,9 +38,9 @@ describe('ci-visibility/requests/upload-test-screenshot', () => { ) assert.ok(requestStub.calledOnce) - const { path, headers, deadline, signal } = requestStub.getCall(0).args[1] + const { path, headers, deadline, retryUntilDeadline, signal } = requestStub.getCall(0).args[1] const query = new URL(path, 'http://localhost:8126').searchParams - return { path, headers, query, deadline, signal } + return { path, headers, query, deadline, retryUntilDeadline, signal } } before(() => { @@ -87,6 +87,7 @@ describe('ci-visibility/requests/upload-test-screenshot', () => { const requestOptions = uploadForFile('screenshot.png', { deadline, signal: abortController.signal }) assert.strictEqual(requestOptions.deadline, deadline) + assert.strictEqual(requestOptions.retryUntilDeadline, false) assert.strictEqual(requestOptions.signal, abortController.signal) }) From a544dcd3c4efc158174b3d5bb966b84842109b79 Mon Sep 17 00:00:00 2001 From: Brian Marks Date: Fri, 28 Aug 2026 15:59:31 -0400 Subject: [PATCH 14/99] test(install): pin standardwebhooks for Anthropic fixtures (#10054) * test(install): pin standardwebhooks for Anthropic fixtures standardwebhooks 1.1.0 is CommonJS but depends on ESM-only @stablelib/base64 2, which breaks Node 18 fixture loading. Carry test-version resolutions into generated workspaces and pin standardwebhooks to 1.0.0 until upstream publishes a compatible release. * test(anthropic): pin webhook dependency in ESM sandbox The ESM integration sandbox installs the Anthropic SDK independently of generated version workspaces, so carry the standardwebhooks 1.0.0 pin into that sandbox too. --- .../test/integration-test/client.spec.js | 2 ++ packages/dd-trace/test/plugins/versions/package.json | 3 +++ scripts/install_plugin_modules.js | 3 ++- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/datadog-plugin-anthropic/test/integration-test/client.spec.js b/packages/datadog-plugin-anthropic/test/integration-test/client.spec.js index cc069bab78a..318339409c4 100644 --- a/packages/datadog-plugin-anthropic/test/integration-test/client.spec.js +++ b/packages/datadog-plugin-anthropic/test/integration-test/client.spec.js @@ -14,6 +14,7 @@ const { stopProc, } = require('../../../../integration-tests/helpers') const { withVersions } = require('../../../dd-trace/test/setup/mocha') +const { resolutions } = require('../../../dd-trace/test/plugins/versions/package.json') describe('esm', () => { let agent @@ -22,6 +23,7 @@ describe('esm', () => { withVersions('anthropic', ['@anthropic-ai/sdk'], version => { useSandbox([ `@anthropic-ai/sdk@${version}`, + `standardwebhooks@${resolutions.standardwebhooks}`, ], false, [ './packages/datadog-plugin-anthropic/test/integration-test/*', ]) diff --git a/packages/dd-trace/test/plugins/versions/package.json b/packages/dd-trace/test/plugins/versions/package.json index 148aabec877..c97d3bce23f 100644 --- a/packages/dd-trace/test/plugins/versions/package.json +++ b/packages/dd-trace/test/plugins/versions/package.json @@ -3,6 +3,9 @@ "version": "1.0.0", "license": "BSD-3-Clause", "private": true, + "resolutions": { + "standardwebhooks": "1.0.0" + }, "dependencies": { "@ai-sdk/amazon-bedrock": "5.0.57", "@ai-sdk/anthropic": "4.0.39", diff --git a/scripts/install_plugin_modules.js b/scripts/install_plugin_modules.js index bd0176e1790..3b0c045e434 100644 --- a/scripts/install_plugin_modules.js +++ b/scripts/install_plugin_modules.js @@ -12,7 +12,7 @@ const semver = require('semver') const externals = require('../packages/dd-trace/test/plugins/externals') const { getInstrumentation } = require('../packages/dd-trace/test/setup/helpers/load-inst') const { getCappedRange, resolvePluginVersions } = require('../packages/dd-trace/test/plugins/versions') -const latests = require('../packages/dd-trace/test/plugins/versions/package.json').dependencies +const { dependencies: latests, resolutions } = require('../packages/dd-trace/test/plugins/versions/package.json') const { isRelativeRequire } = require('../packages/datadog-instrumentations/src/helpers/shared-utils') const exec = require('./helpers/exec') const mapWithConcurrency = require('./helpers/concurrency') @@ -354,6 +354,7 @@ async function assertWorkspaces () { version: '1.0.0', license: 'BSD-3-Clause', private: true, + resolutions, workspaces: { packages: [...workspaces].sort(), }, From 33cc4509b90e8ee076260c69cc964f3e50782cc2 Mon Sep 17 00:00:00 2001 From: Pablo Erhard <104538390+pabloerhard@users.noreply.github.com> Date: Fri, 28 Aug 2026 17:49:46 -0400 Subject: [PATCH 15/99] docs(agents): simplify repository guidance (#9969) --- .agents/skills/apm-integrations/SKILL.md | 5 +- .../references/new-integration-guide.md | 10 +- .../references/plugin-patterns.md | 6 +- .../apm-integrations/references/testing.md | 21 +- .agents/skills/architecture-review/SKILL.md | 86 ++++ .agents/skills/flaky-test-fixer/SKILL.md | 108 +++-- .claude/skills/architecture-review | 1 + .claude/skills/flaky-test-fixer | 1 + .claude/skills/serverless-integrations | 1 + .cursor/skills/architecture-review | 1 + .cursor/skills/flaky-test-fixer | 1 + .cursor/skills/llmobs-integration | 1 + .cursor/skills/llmobs-testing | 1 + .cursor/skills/serverless-integrations | 1 + .github/CODEOWNERS | 11 + AGENTS.md | 447 +++++------------- package.json | 4 +- scripts/check-agents-md-size.js | 23 + scripts/check-agents-md-size.spec.mjs | 49 ++ 19 files changed, 408 insertions(+), 370 deletions(-) create mode 100644 .agents/skills/architecture-review/SKILL.md create mode 120000 .claude/skills/architecture-review create mode 120000 .claude/skills/flaky-test-fixer create mode 120000 .claude/skills/serverless-integrations create mode 120000 .cursor/skills/architecture-review create mode 120000 .cursor/skills/flaky-test-fixer create mode 120000 .cursor/skills/llmobs-integration create mode 120000 .cursor/skills/llmobs-testing create mode 120000 .cursor/skills/serverless-integrations create mode 100644 scripts/check-agents-md-size.js create mode 100644 scripts/check-agents-md-size.spec.mjs diff --git a/.agents/skills/apm-integrations/SKILL.md b/.agents/skills/apm-integrations/SKILL.md index 59399cffeb0..1f45ec5e13e 100644 --- a/.agents/skills/apm-integrations/SKILL.md +++ b/.agents/skills/apm-integrations/SKILL.md @@ -165,7 +165,8 @@ Follow these steps when creating or modifying an integration: 1. **Investigate** — Read the upstream library's source (see [Read Upstream Source First](#read-upstream-source-first)). Read 1-2 reference integrations of the same type (see table above). Understand the instrumentation and plugin patterns before writing code. 2. **Implement instrumentation** — Create the instrumentation in `packages/datadog-instrumentations/src/`. Use orchestrion for instrumentation. 3. **Implement plugin** — Create the plugin in `packages/datadog-plugin-/src/`. Extend the correct base class. -4. **Register** — Add entries in `packages/dd-trace/src/plugins/index.js`, `index.d.ts`, `docs/test.ts`, `docs/API.md`, and `.github/workflows/apm-integrations.yml`. +4. **Register** — Add entries in `packages/dd-trace/src/plugins/index.js`, every supported public TypeScript surface, + `docs/test.ts`, `docs/API.md`, and `.github/workflows/apm-integrations.yml`. 5. **Write tests** — Add unit tests and ESM integration tests. See [Testing](references/testing.md) for templates. 6. **Run tests** — Validate with: @@ -176,7 +177,7 @@ Follow these steps when creating or modifying an integration: # If the plugin needs external services (databases, message brokers, etc.), # check docker-compose.yml for available service names, then: docker compose up -d - PLUGINS="" npm run test:plugins:ci + SERVICES="" PLUGINS="" npm run test:plugins:ci ``` 7. **Verify** — Confirm all tests pass before marking work as complete. diff --git a/.agents/skills/apm-integrations/references/new-integration-guide.md b/.agents/skills/apm-integrations/references/new-integration-guide.md index 30fcefb8e34..b93e868fad3 100644 --- a/.agents/skills/apm-integrations/references/new-integration-guide.md +++ b/.agents/skills/apm-integrations/references/new-integration-guide.md @@ -177,6 +177,7 @@ class MyPlugin extends DatabasePlugin { // Orchestrion: static prefix = 'tracing:orchestrion::' // Shimmer + tracingChannel: static prefix = 'tracing:apm::' // Shimmer + manual channels: omit prefix — defaults to `apm:${id}:${operation}` + static prefix = '' static peerServicePrecursors = ['db.name'] bindStart (ctx) { @@ -196,6 +197,11 @@ class MyPlugin extends DatabasePlugin { return ctx.currentStore } + + // Choose `end` (sync), `asyncEnd` (promise/callback), or `finish` (legacy manual channel). + asyncEnd (ctx) { + this.finish(ctx) + } } module.exports = MyPlugin @@ -215,7 +221,7 @@ If multiple npm packages map to the same plugin (e.g., `redis` and `@redis/clien ## Step 4: Add TypeScript Definitions -In `index.d.ts`, add to the `plugins` namespace: +Add the plugin type to the `plugins` namespace in every supported public TypeScript surface: ```typescript // In the Plugins interface: @@ -296,7 +302,7 @@ PLUGINS="" npm run test:plugins:ci - [ ] Registered in hooks.js (required for both orchestrion and shimmer paths) - [ ] Plugin created with correct base class - [ ] Plugin registered in `packages/dd-trace/src/plugins/index.js` -- [ ] TypeScript definitions added to `index.d.ts` +- [ ] TypeScript definitions added to every supported public TypeScript surface - [ ] Type check added to `docs/test.ts` - [ ] Documentation added to `docs/API.md` - [ ] CI job added to `.github/workflows/apm-integrations.yml` diff --git a/.agents/skills/apm-integrations/references/plugin-patterns.md b/.agents/skills/apm-integrations/references/plugin-patterns.md index 8a14a796503..047175c7afc 100644 --- a/.agents/skills/apm-integrations/references/plugin-patterns.md +++ b/.agents/skills/apm-integrations/references/plugin-patterns.md @@ -14,7 +14,7 @@ The channel prefix is determined by the instrumentation type. Node.js `tracingCh When using shimmer, prefer `tracingChannel` over manual channels — it provides `start/end/asyncStart/asyncEnd/error` events automatically, consistent with how orchestrion works internally. -This means the plugin only needs to define static properties and implement `bindStart`: +This asynchronous Orchestrion example creates the span in `bindStart` and finishes it in `asyncEnd`: ### Orchestrion Plugin (preferred) ```javascript @@ -29,6 +29,10 @@ class MyPlugin extends TracingPlugin { }, ctx) return ctx.currentStore } + + asyncEnd (ctx) { + this.finish(ctx) + } } ``` diff --git a/.agents/skills/apm-integrations/references/testing.md b/.agents/skills/apm-integrations/references/testing.md index d500d0d52a0..8f4ca3ac06c 100644 --- a/.agents/skills/apm-integrations/references/testing.md +++ b/.agents/skills/apm-integrations/references/testing.md @@ -144,7 +144,6 @@ const { withVersions } = require('../../../dd-trace/test/setup/mocha') describe('esm', () => { let agent let proc - let variants withVersions('', '', version => { useSandbox([`'@${version}'`], false, [ @@ -154,8 +153,12 @@ describe('esm', () => { agent = await new FakeAgent().start() }) - before(async function () { - variants = varySandbox('server.mjs', '', '') + const variants = varySandbox('server.mjs', { + bindingName: 'myLib', + packageName: '', + defaultExport: true, + namedExports: [''], + namedExportBinding: 'namespace', }) afterEach(async () => { @@ -163,7 +166,7 @@ describe('esm', () => { await agent.stop() }) - for (const variant of varySandbox.VARIANTS) { + for (const variant of Object.keys(variants)) { it(`is instrumented ${variant}`, async () => { const res = agent.assertMessageReceived(({ headers, payload }) => { assert.strictEqual(headers.host, `127.0.0.1:${agent.port}`) @@ -182,9 +185,9 @@ describe('esm', () => { ### Key ESM Test Concepts -- `varySandbox(filename, bindingName, namedExport, packageName, byPassDefault)` generates three import-style variants (default, star, destructure) to verify all ESM import patterns -- `varySandbox.VARIANTS` is `['default', 'star', 'destructure']` -- Pass `byPassDefault: true` as fifth argument when the module has no default export +- `varySandbox(filename, options)` generates the import variants supported by the package's export shape. +- Set `defaultExport`, `namedExports`, and `namedExportBinding` from the installed package's real exports. +- Iterate over `Object.keys(variants)`; the returned object maps each generated variant to its filename. - `useSandbox` installs package versions into a temp sandbox directory - `spawnPluginIntegrationTestProcAndExpectExit` spawns `node