From a3ea010785c5e0c74672a04e32ed9c4f141c2fad Mon Sep 17 00:00:00 2001 From: Olivier Notteghem Date: Tue, 14 Apr 2026 15:16:21 -0700 Subject: [PATCH] [Android] Add --mnemonic_cache_salt flag for per-action-type cache invalidation Add a new `--mnemonic_cache_salt==` flag that allows selectively busting the local and remote action cache for specific action types (e.g. `--mnemonic_cache_salt=Javac=V2`). The salt is included in the action properties digest (local cache) and in the CacheSalt proto (remote cache), so changing or removing it forces a cache miss only for the targeted mnemonic. Salts for unrelated mnemonics have no effect on other actions. Changes: - ExecutionOptions: add `--mnemonic_cache_salt` flag + `getMnemonicCacheSalts()` helper - ActionCache.Entry: include optional mnemonic salt in `digestActionProperties()` - ActionCacheChecker: thread `mnemonicCacheSalts` through `getTokenIfNeedToExecute()` and `updateActionCache()`; add backward-compat overloads - SkyframeActionExecutor / SkyframeExecutor: accept and store `mnemonicCacheSalts` at configure time - SkyframeBuilder / ExecutionTool: read the flag and pass it down through the builder pipeline - RemoteExecutionService: include mnemonic salt in `CacheSalt` proto for remote cache - RemoteActionContextProvider: wire `mnemonicCacheSalts` into `RemoteExecutionService` via setter - cache_salt.proto: add `mnemonic_salt` field (field 4) - Tests: fix pre-existing 8-param `updateActionCache` call; add 7 new tests covering salt hit/miss semantics Co-Authored-By: Claude Sonnet 4.6 --- .../build/lib/actions/ActionCacheChecker.java | 97 +++++++++++++-- .../build/lib/actions/cache/ActionCache.java | 24 +++- .../build/lib/buildtool/ExecutionTool.java | 11 +- .../build/lib/buildtool/SkyframeBuilder.java | 27 ++++- .../build/lib/exec/ExecutionOptions.java | 31 +++++ .../remote/RemoteActionContextProvider.java | 6 + .../lib/remote/RemoteExecutionService.java | 16 ++- .../lib/skyframe/SkyframeActionExecutor.java | 18 ++- .../build/lib/skyframe/SkyframeExecutor.java | 11 +- .../devtools/build/lib/vfs/DigestUtils.java | 11 ++ src/main/protobuf/cache_salt.proto | 4 + .../lib/actions/ActionCacheCheckerTest.java | 112 ++++++++++++++++++ .../google/devtools/build/lib/actions/BUILD | 1 + 13 files changed, 349 insertions(+), 20 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionCacheChecker.java b/src/main/java/com/google/devtools/build/lib/actions/ActionCacheChecker.java index 068dea8f528601..134ba78671493e 100644 --- a/src/main/java/com/google/devtools/build/lib/actions/ActionCacheChecker.java +++ b/src/main/java/com/google/devtools/build/lib/actions/ActionCacheChecker.java @@ -559,7 +559,8 @@ public Token getTokenIfNeedToExecute( OutputMetadataStore outputMetadataStore, ArtifactExpander artifactExpander, Map remoteDefaultPlatformProperties, - @Nullable RemoteArtifactChecker remoteArtifactChecker) + @Nullable RemoteArtifactChecker remoteArtifactChecker, + ImmutableMap mnemonicCacheSalts) throws InterruptedException { // TODO(bazel-team): (2010) For RunfilesAction/SymlinkAction and similar actions that // produce only symlinks we should not check whether inputs are valid at all - all that matters @@ -618,7 +619,8 @@ public Token getTokenIfNeedToExecute( outputPermissions, remoteDefaultPlatformProperties, cachedOutputMetadata, - remoteArtifactChecker)) { + remoteArtifactChecker, + mnemonicCacheSalts.get(action.getMnemonic()))) { if (entry != null) { removeCacheEntry(action); } @@ -638,6 +640,34 @@ public Token getTokenIfNeedToExecute( return null; } + /** Backward-compat overload for callers that don't pass mnemonicCacheSalts. */ + @Nullable + public Token getTokenIfNeedToExecute( + Action action, + List resolvedCacheArtifacts, + Map clientEnv, + OutputPermissions outputPermissions, + EventHandler handler, + InputMetadataProvider inputMetadataProvider, + OutputMetadataStore outputMetadataStore, + ArtifactExpander artifactExpander, + Map remoteDefaultPlatformProperties, + @Nullable RemoteArtifactChecker remoteArtifactChecker) + throws InterruptedException { + return getTokenIfNeedToExecute( + action, + resolvedCacheArtifacts, + clientEnv, + outputPermissions, + handler, + inputMetadataProvider, + outputMetadataStore, + artifactExpander, + remoteDefaultPlatformProperties, + remoteArtifactChecker, + ImmutableMap.of()); + } + private boolean mustExecute( Action action, @Nullable ActionCache.Entry entry, @@ -651,7 +681,8 @@ private boolean mustExecute( OutputPermissions outputPermissions, Map remoteDefaultPlatformProperties, @Nullable CachedOutputMetadata cachedOutputMetadata, - @Nullable RemoteArtifactChecker remoteArtifactChecker) + @Nullable RemoteArtifactChecker remoteArtifactChecker, + @Nullable String mnemonicSalt) throws InterruptedException { // Unconditional execution can be applied only for actions that are allowed to be executed. if (unconditionalExecution(action)) { @@ -696,7 +727,7 @@ private boolean mustExecute( Map usedEnvironment = computeUsedEnv(action, clientEnv, remoteDefaultPlatformProperties); - if (!entry.sameActionProperties(usedEnvironment, outputPermissions)) { + if (!entry.sameActionProperties(usedEnvironment, outputPermissions, mnemonicSalt)) { reportClientEnv(handler, action, usedEnvironment); actionCache.accountMiss(MissReason.DIFFERENT_ENVIRONMENT); return true; @@ -771,7 +802,8 @@ public void updateActionCache( Map clientEnv, OutputPermissions outputPermissions, Map remoteDefaultPlatformProperties, - boolean isDelayedUpdate) + boolean isDelayedUpdate, + ImmutableMap mnemonicCacheSalts) throws IOException, InterruptedException { checkState(cacheConfig.enabled(), "cache unexpectedly disabled, action: %s", action); Preconditions.checkArgument(token != null, "token unexpectedly null, action: %s", action); @@ -797,7 +829,8 @@ public void updateActionCache( ActionCache.Entry entry = new ActionCache.Entry( - actionKey, usedEnvironment, action.discoversInputs(), outputPermissions); + actionKey, usedEnvironment, action.discoversInputs(), outputPermissions, + mnemonicCacheSalts.get(action.getMnemonic())); for (Artifact output : action.getOutputs()) { // Remove old records from the cache if they used different key. String execPath = output.getExecPathString(); @@ -864,7 +897,8 @@ public void updateActionCache( clientEnv, outputPermissions, remoteDefaultPlatformProperties, - true /* isDelayedUpdate */ + true /* isDelayedUpdate */, + mnemonicCacheSalts ); } } @@ -881,6 +915,55 @@ public void updateActionCache( } } + /** Backward-compat overload for callers that don't pass mnemonicCacheSalts. */ + public void updateActionCache( + Action action, + Token token, + InputMetadataProvider inputMetadataProvider, + OutputMetadataStore outputMetadataStore, + ArtifactExpander artifactExpander, + Map clientEnv, + OutputPermissions outputPermissions, + Map remoteDefaultPlatformProperties, + boolean isDelayedUpdate) + throws IOException, InterruptedException { + updateActionCache( + action, + token, + inputMetadataProvider, + outputMetadataStore, + artifactExpander, + clientEnv, + outputPermissions, + remoteDefaultPlatformProperties, + isDelayedUpdate, + ImmutableMap.of()); + } + + /** Backward-compat overload for callers that don't pass isDelayedUpdate or mnemonicCacheSalts. */ + public void updateActionCache( + Action action, + Token token, + InputMetadataProvider inputMetadataProvider, + OutputMetadataStore outputMetadataStore, + ArtifactExpander artifactExpander, + Map clientEnv, + OutputPermissions outputPermissions, + Map remoteDefaultPlatformProperties) + throws IOException, InterruptedException { + updateActionCache( + action, + token, + inputMetadataProvider, + outputMetadataStore, + artifactExpander, + clientEnv, + outputPermissions, + remoteDefaultPlatformProperties, + /* isDelayedUpdate= */ false, + ImmutableMap.of()); + } + @Nullable public List getCachedInputs(Action action, PackageRootResolver resolver) throws PackageRootResolver.PackageRootException, InterruptedException { diff --git a/src/main/java/com/google/devtools/build/lib/actions/cache/ActionCache.java b/src/main/java/com/google/devtools/build/lib/actions/cache/ActionCache.java index 85ee3ec6b1b181..1766fd5eec4211 100644 --- a/src/main/java/com/google/devtools/build/lib/actions/cache/ActionCache.java +++ b/src/main/java/com/google/devtools/build/lib/actions/cache/ActionCache.java @@ -176,8 +176,17 @@ public Entry( Map usedClientEnv, boolean discoversInputs, OutputPermissions outputPermissions) { + this(key, usedClientEnv, discoversInputs, outputPermissions, null); + } + + public Entry( + String key, + Map usedClientEnv, + boolean discoversInputs, + OutputPermissions outputPermissions, + @Nullable String mnemonicSalt) { actionKey = key; - this.actionPropertiesDigest = digestActionProperties(usedClientEnv, outputPermissions); + this.actionPropertiesDigest = digestActionProperties(usedClientEnv, outputPermissions, mnemonicSalt); files = discoversInputs ? new ArrayList<>() : null; mdMap = new HashMap<>(); outputFileMetadata = new HashMap<>(); @@ -206,6 +215,11 @@ public Entry( */ private static byte[] digestActionProperties( Map clientEnv, OutputPermissions outputPermissions) { + return digestActionProperties(clientEnv, outputPermissions, null); + } + + private static byte[] digestActionProperties( + Map clientEnv, OutputPermissions outputPermissions, @Nullable String mnemonicSalt) { byte[] result = EMPTY_CLIENT_ENV_DIGEST; Fingerprint fp = new Fingerprint(); for (Map.Entry entry : clientEnv.entrySet()) { @@ -220,6 +234,10 @@ private static byte[] digestActionProperties( fp.addInt(outputPermissions.getPermissionsMode()); result = DigestUtils.combineUnordered(result, fp.digestAndReset()); } + if (mnemonicSalt != null) { + fp.addString(mnemonicSalt); + result = DigestUtils.xor(result, fp.digestAndReset()); + } return result; } @@ -323,9 +341,9 @@ public byte[] getActionPropertiesDigest() { /** Determines whether this entry has the same action properties as the one given. */ public boolean sameActionProperties( - Map clientEnv, OutputPermissions outputPermissions) { + Map clientEnv, OutputPermissions outputPermissions, @Nullable String mnemonicSalt) { return Arrays.equals( - digestActionProperties(clientEnv, outputPermissions), actionPropertiesDigest); + digestActionProperties(clientEnv, outputPermissions, mnemonicSalt), actionPropertiesDigest); } /** diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java b/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java index f34204f74deca4..1f48e9c8925bb4 100644 --- a/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java +++ b/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java @@ -340,8 +340,11 @@ void prepareForExecution(Stopwatch executionTimer) remoteArtifactChecker, buildRequestOptions.fsvcThreads); try (SilentCloseable c = Profiler.instance().profile("configureActionExecutor")) { + ExecutionOptions executionOptions = request.getOptions(ExecutionOptions.class); skyframeExecutor.configureActionExecutor( - skyframeBuilder.getFileCache(), skyframeBuilder.getActionInputPrefetcher()); + skyframeBuilder.getFileCache(), + skyframeBuilder.getActionInputPrefetcher(), + executionOptions != null ? executionOptions.getMnemonicCacheSalts() : ImmutableMap.of()); } skyframeExecutor.deleteActionsIfRemoteOptionsChanged(request); @@ -953,6 +956,9 @@ private Builder createBuilder( ModifiedFileSet modifiedOutputFiles, boolean shouldStoreRemoteOutputMetadataInActionCache) { BuildRequestOptions options = request.getBuildOptions(); + ExecutionOptions executionOptions = request.getOptions(ExecutionOptions.class); + ImmutableMap mnemonicCacheSalts = + executionOptions != null ? executionOptions.getMnemonicCacheSalts() : ImmutableMap.of(); skyframeExecutor.setActionOutputRoot(env.getActionTempsDirectory()); @@ -986,7 +992,8 @@ private Builder createBuilder( env.getFileCache(), prefetcher, env.getOutputDirectoryHelper(), - env.getRuntime().getBugReporter()); + env.getRuntime().getBugReporter(), + mnemonicCacheSalts); } @VisibleForTesting diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/SkyframeBuilder.java b/src/main/java/com/google/devtools/build/lib/buildtool/SkyframeBuilder.java index cd75af0de53847..b5b84ce32b87fe 100644 --- a/src/main/java/com/google/devtools/build/lib/buildtool/SkyframeBuilder.java +++ b/src/main/java/com/google/devtools/build/lib/buildtool/SkyframeBuilder.java @@ -18,6 +18,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Range; import com.google.common.collect.Sets; @@ -72,6 +73,7 @@ public class SkyframeBuilder implements Builder { private final ActionOutputDirectoryHelper actionOutputDirectoryHelper; private final ActionCacheChecker actionCacheChecker; private final BugReporter bugReporter; + private final ImmutableMap mnemonicCacheSalts; @VisibleForTesting public SkyframeBuilder( @@ -83,6 +85,28 @@ public SkyframeBuilder( ActionInputPrefetcher actionInputPrefetcher, ActionOutputDirectoryHelper actionOutputDirectoryHelper, BugReporter bugReporter) { + this( + skyframeExecutor, + resourceManager, + actionCacheChecker, + modifiedOutputFiles, + fileCache, + actionInputPrefetcher, + actionOutputDirectoryHelper, + bugReporter, + ImmutableMap.of()); + } + + public SkyframeBuilder( + SkyframeExecutor skyframeExecutor, + ResourceManager resourceManager, + ActionCacheChecker actionCacheChecker, + ModifiedFileSet modifiedOutputFiles, + InputMetadataProvider fileCache, + ActionInputPrefetcher actionInputPrefetcher, + ActionOutputDirectoryHelper actionOutputDirectoryHelper, + BugReporter bugReporter, + ImmutableMap mnemonicCacheSalts) { this.resourceManager = resourceManager; this.skyframeExecutor = skyframeExecutor; this.actionCacheChecker = actionCacheChecker; @@ -91,6 +115,7 @@ public SkyframeBuilder( this.actionInputPrefetcher = actionInputPrefetcher; this.actionOutputDirectoryHelper = actionOutputDirectoryHelper; this.bugReporter = bugReporter; + this.mnemonicCacheSalts = mnemonicCacheSalts; } @Override @@ -115,7 +140,7 @@ public void buildArtifacts( skyframeExecutor.detectModifiedOutputFiles( modifiedOutputFiles, lastExecutionTimeRange, remoteArtifactChecker, fsvcThreads); try (SilentCloseable c = Profiler.instance().profile("configureActionExecutor")) { - skyframeExecutor.configureActionExecutor(fileCache, actionInputPrefetcher); + skyframeExecutor.configureActionExecutor(fileCache, actionInputPrefetcher, mnemonicCacheSalts); } // Note that executionProgressReceiver accesses builtTargets concurrently (after wrapping in a // synchronized collection), so unsynchronized access to this variable is unsafe while it runs. diff --git a/src/main/java/com/google/devtools/build/lib/exec/ExecutionOptions.java b/src/main/java/com/google/devtools/build/lib/exec/ExecutionOptions.java index 811567aea62c74..2aad2eaec715ad 100644 --- a/src/main/java/com/google/devtools/build/lib/exec/ExecutionOptions.java +++ b/src/main/java/com/google/devtools/build/lib/exec/ExecutionOptions.java @@ -13,6 +13,7 @@ // limitations under the License. package com.google.devtools.build.lib.exec; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; import com.google.devtools.build.lib.actions.ActionExecutionContext; import com.google.devtools.build.lib.actions.ActionExecutionContext.ShowSubcommands; @@ -562,6 +563,36 @@ public boolean usingLocalTestJobs() { + " code 39.") public int remoteRetryOnTransientCacheError; + @Option( + name = "mnemonic_cache_salt", + allowMultiple = true, + converter = Converters.AssignmentConverter.class, + defaultValue = "null", + documentationCategory = OptionDocumentationCategory.EXECUTION_STRATEGY, + effectTags = {OptionEffectTag.EXECUTION, OptionEffectTag.ACTION_COMMAND_LINES}, + help = + "Specify a salt value to include in the cache key for actions with a given mnemonic. " + + "Format: --mnemonic_cache_salt=Mnemonic=salt. " + + "This invalidates both local and remote cache entries for that mnemonic. " + + "Example: --mnemonic_cache_salt=Javac=V2 --mnemonic_cache_salt=KotlinCompile=V3") + public List> mnemonicCacheSalts; + + /** + * Returns the mnemonic cache salts as an immutable map. + * + *

If the same mnemonic is specified multiple times, the last value wins. + */ + public ImmutableMap getMnemonicCacheSalts() { + if (mnemonicCacheSalts == null || mnemonicCacheSalts.isEmpty()) { + return ImmutableMap.of(); + } + ImmutableMap.Builder builder = ImmutableMap.builder(); + for (Map.Entry entry : mnemonicCacheSalts) { + builder.put(entry.getKey(), entry.getValue()); + } + return builder.buildKeepingLast(); + } + /** An enum for specifying different formats of test output. */ public enum TestOutputFormat { SUMMARY, // Provide summary output only. diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteActionContextProvider.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteActionContextProvider.java index e07d73278dde93..ae2e483222ac5b 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/RemoteActionContextProvider.java +++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteActionContextProvider.java @@ -191,6 +191,12 @@ private RemoteExecutionService getRemoteExecutionService() { outputService, knownMissingCasDigests); env.getEventBus().register(remoteExecutionService); + ExecutionOptions remoteExecExecutionOptions = + env.getOptions().getOptions(ExecutionOptions.class); + if (remoteExecExecutionOptions != null) { + remoteExecutionService.setMnemonicCacheSalts( + remoteExecExecutionOptions.getMnemonicCacheSalts()); + } } return remoteExecutionService; diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteExecutionService.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteExecutionService.java index 99d7ed8497cee6..519905904227f9 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/RemoteExecutionService.java +++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteExecutionService.java @@ -201,6 +201,7 @@ public class RemoteExecutionService { private final Set knownMissingCasDigests; private Boolean useOutputPaths; + private ImmutableMap mnemonicCacheSalts = ImmutableMap.of(); public RemoteExecutionService( Executor executor, @@ -345,6 +346,11 @@ && shouldUploadLocalResultsToRemoteCache(remoteOptions, spawn.getExecutionInfo() return CachePolicy.create(allowRemoteCache, allowDiskCache); } + /** Sets the per-mnemonic cache salts for targeted cache invalidation. */ + public void setMnemonicCacheSalts(ImmutableMap mnemonicCacheSalts) { + this.mnemonicCacheSalts = mnemonicCacheSalts; + } + /** Returns {@code true} if the spawn may be executed remotely. */ public boolean mayBeExecutedRemotely(Spawn spawn) { return combinedCache instanceof RemoteExecutionCache @@ -506,7 +512,7 @@ public MerkleTree uncachedBuildMerkleTreeVisitor( } @Nullable - private static ByteString buildSalt(Spawn spawn, @Nullable SpawnScrubber spawnScrubber) { + private static ByteString buildSalt(Spawn spawn, @Nullable SpawnScrubber spawnScrubber, ImmutableMap mnemonicCacheSalts) { CacheSalt.Builder saltBuilder = CacheSalt.newBuilder().setMayBeExecutedRemotely(Spawns.mayBeExecutedRemotely(spawn)); @@ -521,6 +527,12 @@ private static ByteString buildSalt(Spawn spawn, @Nullable SpawnScrubber spawnSc CacheSalt.ScrubSalt.newBuilder().setSalt(spawnScrubber.getSalt()).build()); } + // Add mnemonic-specific salt for targeted cache invalidation. + String mnemonicSalt = mnemonicCacheSalts.get(spawn.getMnemonic()); + if (mnemonicSalt != null) { + saltBuilder.setMnemonicSalt(mnemonicSalt); + } + return saltBuilder.build().toByteString(); } @@ -672,7 +684,7 @@ public RemoteAction buildRemoteAction(Spawn spawn, SpawnExecutionContext context platform, context.getTimeout(), Spawns.mayBeCachedRemotely(spawn), - buildSalt(spawn, spawnScrubber)); + buildSalt(spawn, spawnScrubber, mnemonicCacheSalts)); ActionKey actionKey = digestUtil.computeActionKey(action); diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeActionExecutor.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeActionExecutor.java index 8a925d080ad93f..ed9c5919e2d571 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeActionExecutor.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeActionExecutor.java @@ -205,6 +205,7 @@ public void injectTree(SpecialArtifact output, TreeArtifactValue tree) { private boolean freeDiscoveredInputsAfterExecution; private InputMetadataProvider perBuildFileCache; private ActionInputPrefetcher actionInputPrefetcher; + private ImmutableMap mnemonicCacheSalts = ImmutableMap.of(); /** These variables are nulled out between executions. */ @Nullable private ProgressSupplier progressSupplier; @@ -684,7 +685,8 @@ Token checkActionCache( outputMetadataStore, artifactExpander, remoteDefaultProperties, - remoteArtifactChecker); + remoteArtifactChecker, + mnemonicCacheSalts); if (token == null) { boolean eventPosted = false; @@ -793,7 +795,8 @@ void updateActionCache( clientEnv, getOutputPermissions(), remoteDefaultProperties, - false /* isDelayedUpdate */); + false /* isDelayedUpdate */, + mnemonicCacheSalts); } catch (IOException e) { // Skyframe has already done all the filesystem access needed for outputs and swallows // IOExceptions for inputs. So an IOException is impossible here. @@ -954,10 +957,19 @@ private boolean isBuilderAborting() { public void configure( InputMetadataProvider fileCache, ActionInputPrefetcher actionInputPrefetcher, - DiscoveredModulesPruner discoveredModulesPruner) { + DiscoveredModulesPruner discoveredModulesPruner, + ImmutableMap mnemonicCacheSalts) { this.perBuildFileCache = fileCache; this.actionInputPrefetcher = actionInputPrefetcher; this.discoveredModulesPruner = discoveredModulesPruner; + this.mnemonicCacheSalts = mnemonicCacheSalts; + } + + public void configure( + InputMetadataProvider fileCache, + ActionInputPrefetcher actionInputPrefetcher, + DiscoveredModulesPruner discoveredModulesPruner) { + configure(fileCache, actionInputPrefetcher, discoveredModulesPruner, ImmutableMap.of()); } /** diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java index 12f178e138e5d3..7a11ed2e16738c 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java @@ -898,9 +898,16 @@ protected void checkActive() { } public void configureActionExecutor( - InputMetadataProvider fileCache, ActionInputPrefetcher actionInputPrefetcher) { + InputMetadataProvider fileCache, + ActionInputPrefetcher actionInputPrefetcher, + ImmutableMap mnemonicCacheSalts) { skyframeActionExecutor.configure( - fileCache, actionInputPrefetcher, DiscoveredModulesPruner.DEFAULT); + fileCache, actionInputPrefetcher, DiscoveredModulesPruner.DEFAULT, mnemonicCacheSalts); + } + + public void configureActionExecutor( + InputMetadataProvider fileCache, ActionInputPrefetcher actionInputPrefetcher) { + configureActionExecutor(fileCache, actionInputPrefetcher, ImmutableMap.of()); } @ForOverride diff --git a/src/main/java/com/google/devtools/build/lib/vfs/DigestUtils.java b/src/main/java/com/google/devtools/build/lib/vfs/DigestUtils.java index 1de0f908f4b2cb..f894bd4615886b 100644 --- a/src/main/java/com/google/devtools/build/lib/vfs/DigestUtils.java +++ b/src/main/java/com/google/devtools/build/lib/vfs/DigestUtils.java @@ -205,4 +205,15 @@ public static byte[] combineUnordered(byte[] lhs, byte[] rhs) { } return combineUnordered(rhs, lhs); } + + public static byte[] xor(byte[] lhs, byte[] rhs) { + int n = rhs.length; + if (lhs.length >= n) { + for (int i = 0; i < n; i++) { + lhs[i] ^= rhs[i]; + } + return lhs; + } + return xor(rhs, lhs); + } } diff --git a/src/main/protobuf/cache_salt.proto b/src/main/protobuf/cache_salt.proto index a614194a7641a0..26247833c9c339 100644 --- a/src/main/protobuf/cache_salt.proto +++ b/src/main/protobuf/cache_salt.proto @@ -40,4 +40,8 @@ message CacheSalt { // Ensures that a scrubbed spawn can never collide with a non-scrubbed one. // See the documentation for the --experimental_remote_scrub_config flag. ScrubSalt scrub_salt = 3; + + // Per-mnemonic cache salt value for targeted cache invalidation. + // See the documentation for the --mnemonic_cache_salt flag. + string mnemonic_salt = 4; } diff --git a/src/test/java/com/google/devtools/build/lib/actions/ActionCacheCheckerTest.java b/src/test/java/com/google/devtools/build/lib/actions/ActionCacheCheckerTest.java index 528fcfab2dcfd0..1c9a89e136c0c4 100644 --- a/src/test/java/com/google/devtools/build/lib/actions/ActionCacheCheckerTest.java +++ b/src/test/java/com/google/devtools/build/lib/actions/ActionCacheCheckerTest.java @@ -1752,4 +1752,116 @@ public ActionResult execute(ActionExecutionContext actionExecutionContext) { return super.execute(actionExecutionContext); } } + + // ---- Helpers and tests for --mnemonic_cache_salt feature ---- + + /** + * Runs an action through the cache lifecycle with the given mnemonic salt map. Returns true if + * the action needed to execute (cache miss), false if it was a cache hit. + */ + private boolean runActionWithMnemonicSalts( + Action action, ImmutableMap mnemonicCacheSalts) throws Exception { + FakeInputMetadataHandler metadataHandler = new FakeInputMetadataHandler(); + Token token = + cacheChecker.getTokenIfNeedToExecute( + action, + /* resolvedCacheArtifacts= */ null, + /* clientEnv= */ ImmutableMap.of(), + OutputPermissions.READONLY, + /* handler= */ null, + metadataHandler, + metadataHandler, + /* artifactExpander= */ null, + /* remoteDefaultPlatformProperties= */ ImmutableMap.of(), + RemoteArtifactChecker.TRUST_ALL, + mnemonicCacheSalts); + + if (token != null) { + for (Artifact artifact : action.getOutputs()) { + Path path = artifact.getPath(); + filesToDelete.add(path); + Path parent = path.getParentDirectory(); + if (parent != null) { + parent.createDirectoryAndParents(); + } + } + ActionExecutionContext context = mock(ActionExecutionContext.class); + when(context.getOutputMetadataStore()).thenReturn(metadataHandler); + action.execute(context); + cacheChecker.updateActionCache( + action, + token, + metadataHandler, + metadataHandler, + /* artifactExpander= */ null, + /* clientEnv= */ ImmutableMap.of(), + OutputPermissions.READONLY, + /* remoteDefaultPlatformProperties= */ ImmutableMap.of(), + /* isDelayedUpdate= */ false, + mnemonicCacheSalts); + } + return token != null; + } + + @Test + public void testMnemonicCacheSalt_firstRunIsAlwaysCacheMiss() throws Exception { + Action action = new WriteEmptyOutputAction(); + boolean executed = runActionWithMnemonicSalts(action, ImmutableMap.of("Null", "V1")); + assertThat(executed).isTrue(); + } + + @Test + public void testMnemonicCacheSalt_secondRunWithSameSaltIsCacheHit() throws Exception { + Action action = new WriteEmptyOutputAction(); + ImmutableMap salts = ImmutableMap.of("Null", "V1"); + runActionWithMnemonicSalts(action, salts); + boolean executedAgain = runActionWithMnemonicSalts(action, salts); + assertThat(executedAgain).isFalse(); + } + + @Test + public void testMnemonicCacheSalt_changingSaltCausesCacheMiss() throws Exception { + Action action = new WriteEmptyOutputAction(); + runActionWithMnemonicSalts(action, ImmutableMap.of("Null", "V1")); + boolean executedAfterSaltChange = runActionWithMnemonicSalts(action, ImmutableMap.of("Null", "V2")); + assertThat(executedAfterSaltChange).isTrue(); + } + + @Test + public void testMnemonicCacheSalt_removingSaltCausesCacheMiss() throws Exception { + Action action = new WriteEmptyOutputAction(); + runActionWithMnemonicSalts(action, ImmutableMap.of("Null", "V1")); + // Remove the salt entirely: different digest → cache miss + boolean executedWithoutSalt = runActionWithMnemonicSalts(action, ImmutableMap.of()); + assertThat(executedWithoutSalt).isTrue(); + } + + @Test + public void testMnemonicCacheSalt_addingSaltCausesCacheMiss() throws Exception { + Action action = new WriteEmptyOutputAction(); + // First run with no salt + runActionWithMnemonicSalts(action, ImmutableMap.of()); + // Adding a salt should bust the cache + boolean executedAfterAddSalt = runActionWithMnemonicSalts(action, ImmutableMap.of("Null", "V1")); + assertThat(executedAfterAddSalt).isTrue(); + } + + @Test + public void testMnemonicCacheSalt_saltForDifferentMnemonicDoesNotAffectAction() throws Exception { + Action action = new WriteEmptyOutputAction(); // mnemonic is "Null" + runActionWithMnemonicSalts(action, ImmutableMap.of("Null", "V1")); + // A salt for "Javac" should not affect a "Null" mnemonic action + boolean executed = runActionWithMnemonicSalts(action, ImmutableMap.of("Null", "V1", "Javac", "X")); + assertThat(executed).isFalse(); + } + + @Test + public void testMnemonicCacheSalt_saltOnlyForDifferentMnemonicDoesNotCauseMiss() throws Exception { + Action action = new WriteEmptyOutputAction(); // mnemonic is "Null" + // Cache with no salt + runActionWithMnemonicSalts(action, ImmutableMap.of()); + // Salt only for "Javac" should not affect "Null" mnemonic action + boolean executed = runActionWithMnemonicSalts(action, ImmutableMap.of("Javac", "V1")); + assertThat(executed).isFalse(); + } } diff --git a/src/test/java/com/google/devtools/build/lib/actions/BUILD b/src/test/java/com/google/devtools/build/lib/actions/BUILD index 28af04afb1a099..760d6b5bedcff4 100644 --- a/src/test/java/com/google/devtools/build/lib/actions/BUILD +++ b/src/test/java/com/google/devtools/build/lib/actions/BUILD @@ -52,6 +52,7 @@ java_library( "//src/main/java/com/google/devtools/build/lib/analysis:config/core_options", "//src/main/java/com/google/devtools/build/lib/analysis/platform", "//src/main/java/com/google/devtools/build/lib/bugreport", + "//src/main/java/com/google/devtools/build/lib:build-request-options", "//src/main/java/com/google/devtools/build/lib/clock", "//src/main/java/com/google/devtools/build/lib/cmdline", "//src/main/java/com/google/devtools/build/lib/collect/nestedset",