in Public/Src/Engine/Scheduler/PipExecutor.cs [2223:2809]
private static async Task<RunnableFromCacheResult> TryCheckProcessRunnableFromCacheAsync(
ProcessRunnablePip processRunnable,
PipExecutionState.PipScopeState state,
CacheableProcess cacheableProcess,
Func<WeakContentFingerprint> computeWeakFingerprint,
BoxRef<PipCacheMissEventData> pipCacheMiss,
BoxRef<ProcessFingerprintComputationEventData> processFingerprintComputationResult,
List<BoxRef<ProcessStrongFingerprintComputationData>> strongFingerprintComputationList,
bool canAugmentWeakFingerprint,
bool isWeakFingerprintAugmented)
{
Contract.Requires(processRunnable != null);
Contract.Requires(cacheableProcess != null);
Contract.Requires(!isWeakFingerprintAugmented || !canAugmentWeakFingerprint);
var operationContext = processRunnable.OperationContext;
var environment = processRunnable.Environment;
var pathTable = environment.Context.PathTable;
Contract.Assume(pathTable != null);
var cache = environment.State.Cache;
Contract.Assume(cache != null);
var content = environment.Cache.ArtifactContentCache;
Contract.Assume(content != null);
var process = cacheableProcess.Process;
int numPathSetsDownloaded = 0, numCacheEntriesVisited = 0;
WeakContentFingerprint weakFingerprint;
bool performedLookupForAugmentedWeakFingerprint = false;
using (operationContext.StartOperation(PipExecutorCounter.ComputeWeakFingerprintDuration))
{
weakFingerprint = computeWeakFingerprint();
if (!isWeakFingerprintAugmented)
{
// Only set the weak fingerprint if it is not an augmented one.
// The reason for this is we want to hide the fact that the weak fingerprint is augmented.
processFingerprintComputationResult.Value.WeakFingerprint = weakFingerprint;
}
}
var result = await innerCheckRunnableFromCacheAsync();
// Update the strong fingerprint computations list
processRunnable.CacheLookupPerfInfo.LogCounters(pipCacheMiss.Value.CacheMissType, numPathSetsDownloaded, numCacheEntriesVisited);
Logger.Log.PipCacheLookupStats(
operationContext,
process.FormattedSemiStableHash,
performedLookupForAugmentedWeakFingerprint,
weakFingerprint.ToString(),
numCacheEntriesVisited,
numPathSetsDownloaded);
return result;
// Extracted local function with main logic for performing the cache lookup.
// This is done to ensure that execution log logging is always done even in cases of early return (namely augmented weak fingerprint cache lookup
// defers to an inner cache lookup and performs an early return of the result)
async Task<RunnableFromCacheResult> innerCheckRunnableFromCacheAsync()
{
// Totally usable descriptor (may additionally require content availability), or null.
RunnableFromCacheResult.CacheHitData cacheHitData = null;
PublishedEntryRefLocality? refLocality;
ObservedInputProcessingResult? maybeUsableProcessingResult = null;
string description = processRunnable.Description;
// Augmented weak fingerprint used for storing cache entry in case of cache miss
WeakContentFingerprint? augmentedWeakFingerprint = null;
BoxRef<PipCacheMissEventData> augmentedWeakFingerprintMiss = null;
if (cacheableProcess.ShouldHaveArtificialMiss())
{
pipCacheMiss.Value.CacheMissType = PipCacheMissType.MissForDescriptorsDueToArtificialMissOptions;
Logger.Log.ScheduleArtificialCacheMiss(operationContext, description);
refLocality = null;
}
else if (cacheableProcess.DisableCacheLookup())
{
// No sense in going into the strong fingerprint lookup if cache lookup is disabled.
pipCacheMiss.Value.CacheMissType = PipCacheMissType.MissForProcessConfiguredUncacheable;
Logger.Log.ScheduleProcessConfiguredUncacheable(operationContext, description);
refLocality = null;
}
else
{
// Chapter 1: Determine Strong Fingerprint
// First, we will evaluate a sequence of (path set, strong fingerprint) pairs.
// Each path set generates a particular strong fingerprint based on local build state (input hashes);
// if we find a pair such that the generated strong fingerprint matches, then we should be able to find
// a usable entry (describing the output hashes, etc.) to replay.
// We will set this to the first usable-looking entry we find, if any.
// Note that we do not bother investigating further pairs if we find an entry-ref that can't be fetched,
// or if the fetched entry refers to content that cannot be found. Both are fairly unusual failures for well-behaved caches.
// So, this is assigned at most once for entry into Chapter 2.
PublishedEntryRef? maybeUsableEntryRef = null;
ObservedPathSet? maybePathSet = null;
// Set if we find a usable entry.
refLocality = null;
using (operationContext.StartOperation(PipExecutorCounter.CheckProcessRunnableFromCacheChapter1DetermineStrongFingerprintDuration))
using (var strongFingerprintCacheWrapper = SchedulerPools.HashFingerprintDataMapPool.GetInstance())
{
// It is common to have many entry refs for the same PathSet, since often path content changes more often than the set of paths
// (i.e., the refs differ by strong fingerprint). We cache the strong fingerprint computation per PathSet; this saves the repeated
// cost of fetching and deserializing the path set, validating access to the paths and finding their content, and computing the overall strong fingerprint.
// For those path sets that are ill-defined for the pip (e.g. inaccessible paths), we use a null marker.
Dictionary<ContentHash, Tuple<BoxRef<ProcessStrongFingerprintComputationData>, ObservedInputProcessingResult, ObservedPathSet>> strongFingerprintCache =
strongFingerprintCacheWrapper.Instance;
foreach (Task<Possible<PublishedEntryRef, Failure>> batchPromise in cache.ListPublishedEntriesByWeakFingerprint(operationContext, weakFingerprint))
{
if (environment.Context.CancellationToken.IsCancellationRequested)
{
break;
}
Possible<PublishedEntryRef> maybeBatch;
using (operationContext.StartOperation(PipExecutorCounter.CacheQueryingWeakFingerprintDuration))
{
maybeBatch = await batchPromise;
}
if (!maybeBatch.Succeeded)
{
Logger.Log.TwoPhaseFailureQueryingWeakFingerprint(
operationContext,
description,
weakFingerprint.ToString(),
maybeBatch.Failure.DescribeIncludingInnerFailures());
continue;
}
PublishedEntryRef entryRef = maybeBatch.Result;
if (entryRef.IgnoreEntry)
{
continue;
}
// Only increment for valid entries
++numCacheEntriesVisited;
// First, we use the path-set component of the entry to compute the strong fingerprint we would accept.
// Note that we often can re-use an already computed strong fingerprint (this wouldn't be needed if instead
// the cache returned (path set, [strong fingerprint 1, strong fingerprint 2, ...])
Tuple<BoxRef<ProcessStrongFingerprintComputationData>, ObservedInputProcessingResult, ObservedPathSet> strongFingerprintComputation;
StrongContentFingerprint? strongFingerprint = null;
if (!strongFingerprintCache.TryGetValue(entryRef.PathSetHash, out strongFingerprintComputation))
{
using (operationContext.StartOperation(PipExecutorCounter.TryLoadPathSetFromContentCacheDuration))
{
maybePathSet = await TryLoadPathSetFromContentCacheAsync(
operationContext,
environment,
description,
weakFingerprint,
entryRef.PathSetHash);
}
++numPathSetsDownloaded;
if (!maybePathSet.HasValue)
{
// Failure reason already logged.
// Poison this path set hash so we don't repeatedly try to retrieve and parse it.
strongFingerprintCache[entryRef.PathSetHash] = null;
continue;
}
var pathSet = maybePathSet.Value;
// Record the most relevant strong fingerprint information, defaulting to information retrieved from cache
BoxRef<ProcessStrongFingerprintComputationData> strongFingerprintComputationData = new ProcessStrongFingerprintComputationData(
pathSet: pathSet,
pathSetHash: entryRef.PathSetHash,
priorStrongFingerprints: new List<StrongContentFingerprint>(1) { entryRef.StrongFingerprint });
strongFingerprintComputationList.Add(strongFingerprintComputationData);
// check if now running with safer options than before (i.e., prior are not strictly safer than current)
var currentUnsafeOptions = state.UnsafeOptions;
var priorUnsafeOptions = pathSet.UnsafeOptions;
if (priorUnsafeOptions.IsLessSafeThan(currentUnsafeOptions))
{
// This path set's options are less safe than our current options so we cannot use it. Just ignore it.
// Poison this path set hash so we don't repeatedly try to retrieve and parse it.
strongFingerprintCache[entryRef.PathSetHash] = null;
continue;
}
(ObservedInputProcessingResult observedInputProcessingResult, StrongContentFingerprint computedStrongFingerprint) =
await TryComputeStrongFingerprintBasedOnPriorObservedPathSetAsync(
operationContext,
environment,
state,
cacheableProcess,
weakFingerprint,
pathSet,
entryRef.PathSetHash);
ObservedInputProcessingStatus processingStatus = observedInputProcessingResult.Status;
switch (processingStatus)
{
case ObservedInputProcessingStatus.Success:
strongFingerprint = computedStrongFingerprint;
Contract.Assume(strongFingerprint.HasValue);
strongFingerprintComputationData.Value = strongFingerprintComputationData.Value.ToSuccessfulResult(
computedStrongFingerprint: computedStrongFingerprint,
observedInputs: observedInputProcessingResult.ObservedInputs.BaseArray);
if (ETWLogger.Log.IsEnabled(EventLevel.Verbose, Keywords.Diagnostics))
{
Logger.Log.TwoPhaseStrongFingerprintComputedForPathSet(
operationContext,
description,
weakFingerprint.ToString(),
entryRef.PathSetHash.ToHex(),
strongFingerprint.Value.ToString());
}
break;
case ObservedInputProcessingStatus.Mismatched:
// This pip can't access some of the paths. We should remember that (the path set may be repeated many times).
strongFingerprint = null;
if (ETWLogger.Log.IsEnabled(EventLevel.Verbose, Keywords.Diagnostics))
{
Logger.Log.TwoPhaseStrongFingerprintUnavailableForPathSet(
operationContext,
description,
weakFingerprint.ToString(),
entryRef.PathSetHash.ToHex());
}
break;
default:
Contract.Assume(operationContext.LoggingContext.ErrorWasLogged);
Contract.Assert(processingStatus == ObservedInputProcessingStatus.Aborted);
// An error has already been logged. We have to bail out and fail the pip.
return null;
}
strongFingerprintCache[entryRef.PathSetHash] = strongFingerprintComputation = Tuple.Create(strongFingerprintComputationData, observedInputProcessingResult, pathSet);
}
else if (strongFingerprintComputation != null)
{
// Add the strong fingerprint to the list of strong fingerprints to be reported
strongFingerprintComputation.Item1.Value.AddPriorStrongFingerprint(entryRef.StrongFingerprint);
// Set the strong fingerprint computed for this path set so it can be compared to the
// prior strong fingerprint for a cache hit/miss
if (strongFingerprintComputation.Item1.Value.Succeeded)
{
strongFingerprint = strongFingerprintComputation.Item1.Value.ComputedStrongFingerprint;
}
}
// Now we might have a strong fingerprint.
if (!strongFingerprint.HasValue)
{
// Recall that 'null' is a special value meaning 'this path set will never work'
continue;
}
if (strongFingerprint.Value == entryRef.StrongFingerprint)
{
// Hit! We will immediately commit to this entry-ref. We will have a cache-hit iff
// the entry can be fetched and (if requested) the referenced content can be loaded.
strongFingerprintComputation.Item1.Value.IsStrongFingerprintHit = true;
maybeUsableEntryRef = entryRef;
// We remember locality (local or remote) for attribution later (e.g. we count remote hits separately from local hits).
refLocality = entryRef.Locality;
// We also remember the processingResult
maybeUsableProcessingResult = strongFingerprintComputation.Item2;
maybePathSet = strongFingerprintComputation.Item3;
Logger.Log.TwoPhaseStrongFingerprintMatched(
operationContext,
description,
strongFingerprint: entryRef.StrongFingerprint.ToString(),
strongFingerprintCacheId: entryRef.OriginatingCache);
environment.ReportCacheDescriptorHit(entryRef.OriginatingCache);
break;
}
else if (canAugmentWeakFingerprint && entryRef.StrongFingerprint == StrongContentFingerprint.AugmentedWeakFingerprintMarker)
{
// The strong fingeprint is the marker fingerprint indicating that computing an augmented weak fingerprint is required.
augmentedWeakFingerprint = new WeakContentFingerprint(strongFingerprint.Value.Hash);
// We want to give priority to the cache look-up with augmented weak fingerprint, and so we gives a new pip cahe miss event.
augmentedWeakFingerprintMiss = new PipCacheMissEventData
{
PipId = processRunnable.PipId,
CacheMissType = PipCacheMissType.Invalid,
};
performedLookupForAugmentedWeakFingerprint = true;
strongFingerprintComputation.Item1.Value.AugmentedWeakFingerprint = augmentedWeakFingerprint;
// Notice this is a recursive call to same method with augmented weak fingerprint but disallowing
// further augmentation
var result = await TryCheckProcessRunnableFromCacheAsync(
processRunnable,
state,
cacheableProcess,
() => augmentedWeakFingerprint.Value,
augmentedWeakFingerprintMiss,
processFingerprintComputationResult,
strongFingerprintComputationList,
canAugmentWeakFingerprint: false,
isWeakFingerprintAugmented: true);
string keepAliveResult = "N/A";
try
{
if (result.CanRunFromCache)
{
// Fetch the augmenting path set entry to keep it alive
// NOTE: This is best-effort so we don't observe the result here. This would
// be a good candidate for incorporate since we don't actually need the cache entry
var fetchAugmentingPathSetEntryResult = await cache.TryGetCacheEntryAsync(
cacheableProcess.Process,
weakFingerprint,
entryRef.PathSetHash,
entryRef.StrongFingerprint);
keepAliveResult = fetchAugmentingPathSetEntryResult.Succeeded
? (fetchAugmentingPathSetEntryResult.Result == null ? "Missing" : "Success")
: fetchAugmentingPathSetEntryResult.Failure.Describe();
return result;
}
}
finally
{
Logger.Log.AugmentedWeakFingerprint(
operationContext,
description,
weakFingerprint: weakFingerprint.ToString(),
augmentedWeakFingerprint: augmentedWeakFingerprint.ToString(),
pathSetHash: entryRef.PathSetHash.ToHex(),
pathCount: maybePathSet?.Paths.Length ?? -1,
keepAliveResult: keepAliveResult);
}
}
if (ETWLogger.Log.IsEnabled(EventLevel.Verbose, Keywords.Diagnostics))
{
Logger.Log.TwoPhaseStrongFingerprintRejected(
operationContext,
description,
pathSetHash: entryRef.PathSetHash.ToHex(),
rejectedStrongFingerprint: entryRef.StrongFingerprint.ToString(),
availableStrongFingerprint: strongFingerprint.Value.ToString());
}
}
}
CacheEntry? maybeUsableCacheEntry = null;
using (operationContext.StartOperation(PipExecutorCounter.CheckProcessRunnableFromCacheChapter2RetrieveCacheEntryDuration))
{
// Chapter 2: Retrieve Cache Entry
// If we found a usable-looking entry-ref, then we should be able to fetch the actual entry (containing metadata, and output hashes).
if (maybeUsableEntryRef.HasValue)
{
PublishedEntryRef usableEntryRef = maybeUsableEntryRef.Value;
// The speed of Chapter2 is basically all just this call to GetContentHashList
Possible<CacheEntry?> entryFetchResult =
await cache.TryGetCacheEntryAsync(
cacheableProcess.Process,
weakFingerprint,
usableEntryRef.PathSetHash,
usableEntryRef.StrongFingerprint);
if (entryFetchResult.Succeeded)
{
if (entryFetchResult.Result != null)
{
maybeUsableCacheEntry = entryFetchResult.Result;
}
else
{
// TryGetCacheEntryAsync indicates a graceful miss by returning a null entry. In general, this is reasonable.
// However, since we tried to fetch an entry just recently mentioned by ListPublishedEntriesByWeakFingerprint,
// this is unusual (unusual enough that we don't bother looking for other (path set, strong fingerprint) pairs.
Logger.Log.TwoPhaseCacheEntryMissing(
operationContext,
description,
weakFingerprint: weakFingerprint.ToString(),
strongFingerprint: maybeUsableEntryRef.Value.StrongFingerprint.ToString());
pipCacheMiss.Value.CacheMissType = PipCacheMissType.MissForCacheEntry;
}
}
else
{
Logger.Log.TwoPhaseFetchingCacheEntryFailed(
operationContext,
description,
maybeUsableEntryRef.Value.StrongFingerprint.ToString(),
entryFetchResult.Failure.DescribeIncludingInnerFailures());
pipCacheMiss.Value.CacheMissType = PipCacheMissType.MissForCacheEntry;
}
}
else
{
// We didn't find a usable ref. We can attribute this as a new fingerprint (no refs checked at all)
// or a mismatch of strong fingerprints (at least one ref checked).
if (numCacheEntriesVisited == 0)
{
pipCacheMiss.Value.CacheMissType = isWeakFingerprintAugmented
? PipCacheMissType.MissForDescriptorsDueToAugmentedWeakFingerprints
: PipCacheMissType.MissForDescriptorsDueToWeakFingerprints;
Logger.Log.TwoPhaseCacheDescriptorMissDueToWeakFingerprint(
operationContext,
description,
weakFingerprint.ToString(),
isWeakFingerprintAugmented);
}
else
{
if (augmentedWeakFingerprintMiss != null)
{
// If we ever check augmented weak fingerprint, then we use its miss reason as the miss type.
// This gives priority to the result from cache look-up with augmented weak fingerprint.
// This also makes the data align with execution because if we ever check augmented weak fingerprint,
// then the weak fingerprint for execution is the augmented one.
pipCacheMiss.Value.CacheMissType = augmentedWeakFingerprintMiss.Value.CacheMissType;
}
else
{
pipCacheMiss.Value.CacheMissType = PipCacheMissType.MissForDescriptorsDueToStrongFingerprints;
Logger.Log.TwoPhaseCacheDescriptorMissDueToStrongFingerprints(
operationContext,
description,
weakFingerprint.ToString(),
isWeakFingerprintAugmented);
}
}
}
}
if (maybeUsableCacheEntry.HasValue)
{
cacheHitData = await TryConvertToRunnableFromCacheResultAsync(
processRunnable,
operationContext,
environment,
state,
cacheableProcess,
refLocality.Value,
description,
weakFingerprint,
maybeUsableEntryRef.Value.PathSetHash,
maybeUsableEntryRef.Value.StrongFingerprint,
maybeUsableCacheEntry,
maybePathSet,
pipCacheMiss);
}
}
RunnableFromCacheResult runnableFromCacheResult;
bool isCacheHit = cacheHitData != null;
if (!isCacheHit)
{
var pathSetCount = strongFingerprintComputationList.Count;
int threshold = processRunnable.Process.AugmentWeakFingerprintPathSetThreshold(environment.Configuration.Cache);
if (augmentedWeakFingerprint == null
&& threshold > 0
&& canAugmentWeakFingerprint
&& pathSetCount >= threshold)
{
// Compute 'weak augmenting' path set with common paths among path sets
ObservedPathSet weakAugmentingPathSet = ExtractPathSetForAugmentingWeakFingerprint(pathTable, environment.Configuration.Cache, process, strongFingerprintComputationList);
var minPathCount = strongFingerprintComputationList.Select(s => s.Value.PathSet.Paths.Length).Min();
var maxPathCount = strongFingerprintComputationList.Select(s => s.Value.PathSet.Paths.Length).Max();
var weakAugmentingPathSetHashResult = await cache.TryStorePathSetAsync(weakAugmentingPathSet, processRunnable.Process.PreservePathSetCasing);
string addAugmentingPathSetResultDescription;
if (weakAugmentingPathSetHashResult.Succeeded)
{
ContentHash weakAugmentingPathSetHash = weakAugmentingPathSetHashResult.Result;
// Optional (not currently implemented): If augmenting path set already exists (race condition), we
// could compute augmented weak fingerprint and perform the cache lookup as above
(ObservedInputProcessingResult observedInputProcessingResult, StrongContentFingerprint computedStrongFingerprint) =
await TryComputeStrongFingerprintBasedOnPriorObservedPathSetAsync(
operationContext,
environment,
state,
cacheableProcess,
weakFingerprint,
weakAugmentingPathSet,
weakAugmentingPathSetHash);
BoxRef<ProcessStrongFingerprintComputationData> strongFingerprintComputation = new ProcessStrongFingerprintComputationData(
weakAugmentingPathSetHash,
new List<StrongContentFingerprint>() { StrongContentFingerprint.AugmentedWeakFingerprintMarker },
weakAugmentingPathSet);
// Add the computation of the augmenting weak fingerprint.
// This addition is particularly useful for fingerprint store based cache miss analysis. Later, post execution (due to cache miss),
// we will send ProcessStrongFingerprintComputationEventData to the fingerprint store. That event data will have the augmented
// weak fingerprint as its weak fingerprint. However, we only want to store the original weak fingerprint. Thus, we need
// to create a mapping from the augmented weak fingerprint to the original one when the ProcessStrongFingerprintComputationEventData
// is sent for the cache look-up. Since, the augmented weak fingerprint is actually the strong fingerprint of the current
// strongFingerprintComputation, then we need to include the computation into the computation list.
strongFingerprintComputationList.Add(strongFingerprintComputation);
if (observedInputProcessingResult.Status == ObservedInputProcessingStatus.Success)
{
// Add marker selector with weak augmenting path set
var addAugmentationResult = await cache.TryPublishCacheEntryAsync(
cacheableProcess.Process,
weakFingerprint,
weakAugmentingPathSetHash,
StrongContentFingerprint.AugmentedWeakFingerprintMarker,
CacheEntry.FromArray((new[] { weakAugmentingPathSetHash }).ToReadOnlyArray(), "AugmentWeakFingerprint"));
addAugmentingPathSetResultDescription = addAugmentationResult.Succeeded
? addAugmentationResult.Result.Status.ToString()
: addAugmentationResult.Failure.Describe();
augmentedWeakFingerprint = new WeakContentFingerprint(computedStrongFingerprint.Hash);
strongFingerprintComputation.Value = strongFingerprintComputation.Value.ToSuccessfulResult(
computedStrongFingerprint,
observedInputProcessingResult.ObservedInputs.BaseArray);
strongFingerprintComputation.Value.AugmentedWeakFingerprint = augmentedWeakFingerprint;
strongFingerprintComputation.Value.IsNewlyPublishedAugmentedWeakFingerprint = true;
}
else
{
addAugmentingPathSetResultDescription = observedInputProcessingResult.Status.ToString();
}
}
else
{
addAugmentingPathSetResultDescription = weakAugmentingPathSetHashResult.Failure.Describe();
}
Logger.Log.AddAugmentingPathSet(
operationContext,
cacheableProcess.Description,
weakFingerprint: weakFingerprint.ToString(),
pathSetHash: weakAugmentingPathSetHashResult.Succeeded ? weakAugmentingPathSetHashResult.Result.ToHex() : "N/A",
pathCount: weakAugmentingPathSet.Paths.Length,
pathSetCount: pathSetCount,
minPathCount: minPathCount,
maxPathCount: maxPathCount,
result: addAugmentingPathSetResultDescription);
}
}
WeakContentFingerprint cacheResultWeakFingerprint = isCacheHit || augmentedWeakFingerprint == null
? weakFingerprint
: augmentedWeakFingerprint.Value;
runnableFromCacheResult = CreateRunnableFromCacheResult(
cacheHitData,
environment,
refLocality,
maybeUsableProcessingResult,
cacheResultWeakFingerprint);
return runnableFromCacheResult;
}
}