Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -559,7 +559,8 @@ public Token getTokenIfNeedToExecute(
OutputMetadataStore outputMetadataStore,
ArtifactExpander artifactExpander,
Map<String, String> remoteDefaultPlatformProperties,
@Nullable RemoteArtifactChecker remoteArtifactChecker)
@Nullable RemoteArtifactChecker remoteArtifactChecker,
ImmutableMap<String, String> mnemonicCacheSalts)
throws InterruptedException {
// TODO(bazel-team): (2010) For RunfilesAction/SymlinkAction and similar actions that
// produce only symlinks we should not check whether inputs are valid at all - all that matters
Expand Down Expand Up @@ -618,7 +619,8 @@ public Token getTokenIfNeedToExecute(
outputPermissions,
remoteDefaultPlatformProperties,
cachedOutputMetadata,
remoteArtifactChecker)) {
remoteArtifactChecker,
mnemonicCacheSalts.get(action.getMnemonic()))) {
if (entry != null) {
removeCacheEntry(action);
}
Expand All @@ -638,6 +640,34 @@ public Token getTokenIfNeedToExecute(
return null;
}

/** Backward-compat overload for callers that don't pass mnemonicCacheSalts. */
@Nullable
public Token getTokenIfNeedToExecute(
Action action,
List<Artifact> resolvedCacheArtifacts,
Map<String, String> clientEnv,
OutputPermissions outputPermissions,
EventHandler handler,
InputMetadataProvider inputMetadataProvider,
OutputMetadataStore outputMetadataStore,
ArtifactExpander artifactExpander,
Map<String, String> remoteDefaultPlatformProperties,
@Nullable RemoteArtifactChecker remoteArtifactChecker)
throws InterruptedException {
return getTokenIfNeedToExecute(
action,
resolvedCacheArtifacts,
clientEnv,
outputPermissions,
handler,
inputMetadataProvider,
outputMetadataStore,
artifactExpander,
remoteDefaultPlatformProperties,
remoteArtifactChecker,
ImmutableMap.of());
}

private boolean mustExecute(
Action action,
@Nullable ActionCache.Entry entry,
Expand All @@ -651,7 +681,8 @@ private boolean mustExecute(
OutputPermissions outputPermissions,
Map<String, String> remoteDefaultPlatformProperties,
@Nullable CachedOutputMetadata cachedOutputMetadata,
@Nullable RemoteArtifactChecker remoteArtifactChecker)
@Nullable RemoteArtifactChecker remoteArtifactChecker,
@Nullable String mnemonicSalt)
throws InterruptedException {
// Unconditional execution can be applied only for actions that are allowed to be executed.
if (unconditionalExecution(action)) {
Expand Down Expand Up @@ -696,7 +727,7 @@ private boolean mustExecute(

Map<String, String> usedEnvironment =
computeUsedEnv(action, clientEnv, remoteDefaultPlatformProperties);
if (!entry.sameActionProperties(usedEnvironment, outputPermissions)) {
if (!entry.sameActionProperties(usedEnvironment, outputPermissions, mnemonicSalt)) {
reportClientEnv(handler, action, usedEnvironment);
actionCache.accountMiss(MissReason.DIFFERENT_ENVIRONMENT);
return true;
Expand Down Expand Up @@ -771,7 +802,8 @@ public void updateActionCache(
Map<String, String> clientEnv,
OutputPermissions outputPermissions,
Map<String, String> remoteDefaultPlatformProperties,
boolean isDelayedUpdate)
boolean isDelayedUpdate,
ImmutableMap<String, String> mnemonicCacheSalts)
throws IOException, InterruptedException {
checkState(cacheConfig.enabled(), "cache unexpectedly disabled, action: %s", action);
Preconditions.checkArgument(token != null, "token unexpectedly null, action: %s", action);
Expand All @@ -797,7 +829,8 @@ public void updateActionCache(

ActionCache.Entry entry =
new ActionCache.Entry(
actionKey, usedEnvironment, action.discoversInputs(), outputPermissions);
actionKey, usedEnvironment, action.discoversInputs(), outputPermissions,
mnemonicCacheSalts.get(action.getMnemonic()));
for (Artifact output : action.getOutputs()) {
// Remove old records from the cache if they used different key.
String execPath = output.getExecPathString();
Expand Down Expand Up @@ -864,7 +897,8 @@ public void updateActionCache(
clientEnv,
outputPermissions,
remoteDefaultPlatformProperties,
true /* isDelayedUpdate */
true /* isDelayedUpdate */,
mnemonicCacheSalts
);
}
}
Expand All @@ -881,6 +915,55 @@ public void updateActionCache(
}
}

/** Backward-compat overload for callers that don't pass mnemonicCacheSalts. */
public void updateActionCache(
Action action,
Token token,
InputMetadataProvider inputMetadataProvider,
OutputMetadataStore outputMetadataStore,
ArtifactExpander artifactExpander,
Map<String, String> clientEnv,
OutputPermissions outputPermissions,
Map<String, String> remoteDefaultPlatformProperties,
boolean isDelayedUpdate)
throws IOException, InterruptedException {
updateActionCache(
action,
token,
inputMetadataProvider,
outputMetadataStore,
artifactExpander,
clientEnv,
outputPermissions,
remoteDefaultPlatformProperties,
isDelayedUpdate,
ImmutableMap.of());
}

/** Backward-compat overload for callers that don't pass isDelayedUpdate or mnemonicCacheSalts. */
public void updateActionCache(
Action action,
Token token,
InputMetadataProvider inputMetadataProvider,
OutputMetadataStore outputMetadataStore,
ArtifactExpander artifactExpander,
Map<String, String> clientEnv,
OutputPermissions outputPermissions,
Map<String, String> remoteDefaultPlatformProperties)
throws IOException, InterruptedException {
updateActionCache(
action,
token,
inputMetadataProvider,
outputMetadataStore,
artifactExpander,
clientEnv,
outputPermissions,
remoteDefaultPlatformProperties,
/* isDelayedUpdate= */ false,
ImmutableMap.of());
}

@Nullable
public List<Artifact> getCachedInputs(Action action, PackageRootResolver resolver)
throws PackageRootResolver.PackageRootException, InterruptedException {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -176,8 +176,17 @@ public Entry(
Map<String, String> usedClientEnv,
boolean discoversInputs,
OutputPermissions outputPermissions) {
this(key, usedClientEnv, discoversInputs, outputPermissions, null);
}

public Entry(
String key,
Map<String, String> usedClientEnv,
boolean discoversInputs,
OutputPermissions outputPermissions,
@Nullable String mnemonicSalt) {
actionKey = key;
this.actionPropertiesDigest = digestActionProperties(usedClientEnv, outputPermissions);
this.actionPropertiesDigest = digestActionProperties(usedClientEnv, outputPermissions, mnemonicSalt);
files = discoversInputs ? new ArrayList<>() : null;
mdMap = new HashMap<>();
outputFileMetadata = new HashMap<>();
Expand Down Expand Up @@ -206,6 +215,11 @@ public Entry(
*/
private static byte[] digestActionProperties(
Map<String, String> clientEnv, OutputPermissions outputPermissions) {
return digestActionProperties(clientEnv, outputPermissions, null);
}

private static byte[] digestActionProperties(
Map<String, String> clientEnv, OutputPermissions outputPermissions, @Nullable String mnemonicSalt) {
byte[] result = EMPTY_CLIENT_ENV_DIGEST;
Fingerprint fp = new Fingerprint();
for (Map.Entry<String, String> entry : clientEnv.entrySet()) {
Expand All @@ -220,6 +234,10 @@ private static byte[] digestActionProperties(
fp.addInt(outputPermissions.getPermissionsMode());
result = DigestUtils.combineUnordered(result, fp.digestAndReset());
}
if (mnemonicSalt != null) {
fp.addString(mnemonicSalt);
result = DigestUtils.xor(result, fp.digestAndReset());
}
return result;
}

Expand Down Expand Up @@ -323,9 +341,9 @@ public byte[] getActionPropertiesDigest() {

/** Determines whether this entry has the same action properties as the one given. */
public boolean sameActionProperties(
Map<String, String> clientEnv, OutputPermissions outputPermissions) {
Map<String, String> clientEnv, OutputPermissions outputPermissions, @Nullable String mnemonicSalt) {
return Arrays.equals(
digestActionProperties(clientEnv, outputPermissions), actionPropertiesDigest);
digestActionProperties(clientEnv, outputPermissions, mnemonicSalt), actionPropertiesDigest);
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -340,8 +340,11 @@ void prepareForExecution(Stopwatch executionTimer)
remoteArtifactChecker,
buildRequestOptions.fsvcThreads);
try (SilentCloseable c = Profiler.instance().profile("configureActionExecutor")) {
ExecutionOptions executionOptions = request.getOptions(ExecutionOptions.class);
skyframeExecutor.configureActionExecutor(
skyframeBuilder.getFileCache(), skyframeBuilder.getActionInputPrefetcher());
skyframeBuilder.getFileCache(),
skyframeBuilder.getActionInputPrefetcher(),
executionOptions != null ? executionOptions.getMnemonicCacheSalts() : ImmutableMap.of());
}

skyframeExecutor.deleteActionsIfRemoteOptionsChanged(request);
Expand Down Expand Up @@ -953,6 +956,9 @@ private Builder createBuilder(
ModifiedFileSet modifiedOutputFiles,
boolean shouldStoreRemoteOutputMetadataInActionCache) {
BuildRequestOptions options = request.getBuildOptions();
ExecutionOptions executionOptions = request.getOptions(ExecutionOptions.class);
ImmutableMap<String, String> mnemonicCacheSalts =
executionOptions != null ? executionOptions.getMnemonicCacheSalts() : ImmutableMap.of();

skyframeExecutor.setActionOutputRoot(env.getActionTempsDirectory());

Expand Down Expand Up @@ -986,7 +992,8 @@ private Builder createBuilder(
env.getFileCache(),
prefetcher,
env.getOutputDirectoryHelper(),
env.getRuntime().getBugReporter());
env.getRuntime().getBugReporter(),
mnemonicCacheSalts);
}

@VisibleForTesting
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@

import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Range;
import com.google.common.collect.Sets;
Expand Down Expand Up @@ -72,6 +73,7 @@ public class SkyframeBuilder implements Builder {
private final ActionOutputDirectoryHelper actionOutputDirectoryHelper;
private final ActionCacheChecker actionCacheChecker;
private final BugReporter bugReporter;
private final ImmutableMap<String, String> mnemonicCacheSalts;

@VisibleForTesting
public SkyframeBuilder(
Expand All @@ -83,6 +85,28 @@ public SkyframeBuilder(
ActionInputPrefetcher actionInputPrefetcher,
ActionOutputDirectoryHelper actionOutputDirectoryHelper,
BugReporter bugReporter) {
this(
skyframeExecutor,
resourceManager,
actionCacheChecker,
modifiedOutputFiles,
fileCache,
actionInputPrefetcher,
actionOutputDirectoryHelper,
bugReporter,
ImmutableMap.of());
}

public SkyframeBuilder(
SkyframeExecutor skyframeExecutor,
ResourceManager resourceManager,
ActionCacheChecker actionCacheChecker,
ModifiedFileSet modifiedOutputFiles,
InputMetadataProvider fileCache,
ActionInputPrefetcher actionInputPrefetcher,
ActionOutputDirectoryHelper actionOutputDirectoryHelper,
BugReporter bugReporter,
ImmutableMap<String, String> mnemonicCacheSalts) {
this.resourceManager = resourceManager;
this.skyframeExecutor = skyframeExecutor;
this.actionCacheChecker = actionCacheChecker;
Expand All @@ -91,6 +115,7 @@ public SkyframeBuilder(
this.actionInputPrefetcher = actionInputPrefetcher;
this.actionOutputDirectoryHelper = actionOutputDirectoryHelper;
this.bugReporter = bugReporter;
this.mnemonicCacheSalts = mnemonicCacheSalts;
}

@Override
Expand All @@ -115,7 +140,7 @@ public void buildArtifacts(
skyframeExecutor.detectModifiedOutputFiles(
modifiedOutputFiles, lastExecutionTimeRange, remoteArtifactChecker, fsvcThreads);
try (SilentCloseable c = Profiler.instance().profile("configureActionExecutor")) {
skyframeExecutor.configureActionExecutor(fileCache, actionInputPrefetcher);
skyframeExecutor.configureActionExecutor(fileCache, actionInputPrefetcher, mnemonicCacheSalts);
}
// Note that executionProgressReceiver accesses builtTargets concurrently (after wrapping in a
// synchronized collection), so unsynchronized access to this variable is unsafe while it runs.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
// limitations under the License.
package com.google.devtools.build.lib.exec;

import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
import com.google.devtools.build.lib.actions.ActionExecutionContext;
import com.google.devtools.build.lib.actions.ActionExecutionContext.ShowSubcommands;
Expand Down Expand Up @@ -562,6 +563,36 @@ public boolean usingLocalTestJobs() {
+ " code 39.")
public int remoteRetryOnTransientCacheError;

@Option(
name = "mnemonic_cache_salt",
allowMultiple = true,
converter = Converters.AssignmentConverter.class,
defaultValue = "null",
documentationCategory = OptionDocumentationCategory.EXECUTION_STRATEGY,
effectTags = {OptionEffectTag.EXECUTION, OptionEffectTag.ACTION_COMMAND_LINES},
help =
"Specify a salt value to include in the cache key for actions with a given mnemonic. "
+ "Format: --mnemonic_cache_salt=Mnemonic=salt. "
+ "This invalidates both local and remote cache entries for that mnemonic. "
+ "Example: --mnemonic_cache_salt=Javac=V2 --mnemonic_cache_salt=KotlinCompile=V3")
public List<Map.Entry<String, String>> mnemonicCacheSalts;

/**
* Returns the mnemonic cache salts as an immutable map.
*
* <p>If the same mnemonic is specified multiple times, the last value wins.
*/
public ImmutableMap<String, String> getMnemonicCacheSalts() {
if (mnemonicCacheSalts == null || mnemonicCacheSalts.isEmpty()) {
return ImmutableMap.of();
}
ImmutableMap.Builder<String, String> builder = ImmutableMap.builder();
for (Map.Entry<String, String> entry : mnemonicCacheSalts) {
builder.put(entry.getKey(), entry.getValue());
}
return builder.buildKeepingLast();
}

/** An enum for specifying different formats of test output. */
public enum TestOutputFormat {
SUMMARY, // Provide summary output only.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -191,6 +191,12 @@ private RemoteExecutionService getRemoteExecutionService() {
outputService,
knownMissingCasDigests);
env.getEventBus().register(remoteExecutionService);
ExecutionOptions remoteExecExecutionOptions =
env.getOptions().getOptions(ExecutionOptions.class);
if (remoteExecExecutionOptions != null) {
remoteExecutionService.setMnemonicCacheSalts(
remoteExecExecutionOptions.getMnemonicCacheSalts());
}
}

return remoteExecutionService;
Expand Down
Loading
Loading