in ste/performanceAdvisor.go [158:340]
func (p *PerformanceAdvisor) GetAdvice() []common.PerformanceAdvice {
const (
serverBusyThresholdPercent = 1.0
networkErrorThresholdPercent = 2.0
// we don't have any API to get exact throughput given a VM size. But a quick look at the documentation
// suggests that all current gen VMs get around 500 to 1000 Mbps per core.
// To allow some wriggle room (VM not quite hitting cap, but still affected by that cap) we choose a value a bit less than the min.
// Then, if a VM is getting LESS than this amount per core, we infer that it's not at its throughput cap.
// If it's getting more than this amount AND there are no other constraints, then we assume the constraint IS
// the throughput cap. (It's probably not the network, because if we use this we already know its an Azure VM).
expectedMinAzureMbpsPerCore = 375
// files this size, and smaller, won't trigger the HTBB path on standard storage accounts
htbbThresholdMB = 4
)
result := make([]common.PerformanceAdvice, 0)
// helper functions
alreadyHaveAdvice := func() bool {
return len(result) > 0
}
addAdvice := func(tp AdviceType, reason string, reasonValues ...interface{}) {
reasonText := fmt.Sprintf(reason, reasonValues...)
isFirst := !alreadyHaveAdvice()
result = append(result, common.PerformanceAdvice{
Code: tp.code,
Title: tp.description,
Reason: reasonText,
PriorityAdvice: isFirst})
}
// Note that order matters below - the most important topics are first, and sometimes later ones will
// suppress their output if one of the more important earlier ones has already produced advice.
// Server throttling
haveKnownThrottling := false
if p.serverBusyPercentageIOPS > serverBusyThresholdPercent {
haveKnownThrottling = true
addAdvice(EAdviceType.AccountIOPS(),
"Throughput was throttled by Service on %.0f%% of operations, due to IO Operations Per Second (IOPS) approaching "+
"the limit of your target account. The average IOPS measured in this job was %d operations/sec. "+
" (This message is shown by AzCopy if %.0f%% or more of operations are throttled due to IOPS.)",
p.serverBusyPercentageIOPS, p.iops, serverBusyThresholdPercent)
}
if p.serverBusyPercentageThroughput > serverBusyThresholdPercent {
haveKnownThrottling = true
addAdvice(EAdviceType.AccountThroughput(),
"Throughput was throttled by Service on %.0f%% of operations, due to throughput approaching "+
"the limit of your target account. The throughput measured in this job was %d Mega bits/sec. "+
"(This message is shown by AzCopy if %.0f%% or more of operations are throttled due to throughput.)",
p.serverBusyPercentageThroughput, p.mbps, serverBusyThresholdPercent)
}
if !haveKnownThrottling {
totalThrottling := p.serverBusyPercentageIOPS + p.serverBusyPercentageIOPS + p.serverBusyPercentageOther
if totalThrottling > serverBusyThresholdPercent {
// TODO: reviewers: any improvement on "other reasons" in this text?
addAdvice(EAdviceType.ServerBusy(),
"Throughput was throttled by Service on %.0f%% of operations in total. %.0f%% where throttled due to "+
"account IOPS limits; %.0f%% due to account throughput limits; and %.0f%% for other reasons. "+
"(This message is shown by AzCopy if %.0f%% or more of operations are throttled in total and "+
"no other throttling message applies.)",
totalThrottling, p.serverBusyPercentageIOPS, p.serverBusyPercentageThroughput, p.serverBusyPercentageOther, serverBusyThresholdPercent)
}
}
// Network errors
// (Don't report these if we were throttled by service. Why? Because when service wants to throttle more aggressively,
// it drops connections. We don't want those connection drops, which are actually just severe throttling, to
// be mistaken for actually network problems. So we only report things that look like network problems in cases
// where there was no significant throttling reported)
if !alreadyHaveAdvice() && p.networkErrorPercentage > networkErrorThresholdPercent {
addAdvice(EAdviceType.NetworkErrors(),
"Network errors occurred on %.0f%% of operations. "+
"Network errors include all cases where the HTTP request could not be sent or did not receive a reply. "+
"(This message is shown by AzCopy if %.0f%% or more of operations result in network errors.)",
p.networkErrorPercentage, networkErrorThresholdPercent)
}
// Mbps cap
const mbpsThreshold = 0.9
if p.capMbps > 0 && float32(p.mbps) > mbpsThreshold*float32(p.capMbps) {
addAdvice(EAdviceType.MbpsCapped(),
"Throughput has been capped at %f Mbps with a command line parameter, and the measured throughput was "+
"close to the cap. "+
"(This message is shown by AzCopy if a command-line cap is set and the measured throughput is "+
"over %.0f%% of the cap.)", p.capMbps, mbpsThreshold*100)
}
// Network available bandwidth
if alreadyHaveAdvice() {
addAdvice(EAdviceType.NetworkNotBottleneck(),
"Throughput appears to be limited by other factor(s), so it is assumed that network bandwidth is not the bottleneck. "+
"Throughput of %d Mega bits/sec was obtained with %d concurrent connections.",
p.mbps, p.finalConcurrency)
} else {
switch p.finalConcurrencyTunerReason {
case concurrencyReasonAtOptimum:
isAzureVM := p.azureVmCores > 0
definitelyBelowAzureVMLimit := p.mbps < int64(p.azureVmCores*expectedMinAzureMbpsPerCore)
if isAzureVM &&
p.direction != common.ETransferDirection.S2SCopy() && // VM limits are not relevant in S2S copy
!definitelyBelowAzureVMLimit {
// Azure VM size
// We're not "definitely" below the VM limit, so we _might_ be at it.
// Can't get any more accurate than "might" without specific throughput limit for the VM we're running in,
// and we don't have an API for that
addAdvice(EAdviceType.VMSize(),
"Throughput may be limited by the size of the Azure VM that's running AzCopy. Check the documented expected "+
"network bandwidth for this VM size (%s). If the observed throughput of %d Mbits/sec is close to the documented maximum for the "+
"VM, then upsize to a larger VM. But if the observed throughput is NOT close to the maximum, and your VM is not "+
"in the same region as your target account, consider network bandwidth as a possible bottleneck. (AzCopy displays this "+
"message on Azure VMs when throughput per core is greater than %d Mbps and no other limiting factors were identified.)",
p.azureVmSizeName, p.mbps, expectedMinAzureMbpsPerCore)
// TODO: can we detect if we're in the same region? And can we do any better than that, because in many
// (virtually all?) cases even being in different regions is fine.
} else {
// not limited by VM size, so must be file size, network or Azure Files Standard Share
if p.isToAzureFiles {
// give output that hedges our bets between network and File Share, because we can't tell which is limiting perf
addAdvice(EAdviceType.FileShareOrNetwork(),
"No other factors were identified that are limiting performance, so the bottleneck is assumed to be either "+
"the throughput of the Azure File Share OR the available network bandwidth. To test whether the File Share or the network is "+
"the bottleneck, try running a benchmark to Blob Storage over the same network. If that is much faster, then the bottleneck in this "+
"run was probably the File Share. Check the published Azure File Share throughput targets for more info. In this run throughput "+
"of %d Mega bits/sec was obtained with %d concurrent connections.",
p.mbps, p.finalConcurrency)
} else if p.avgBytesPerFile <= (htbbThresholdMB * 1024 * 1024) {
addAdvice(EAdviceType.SmallFilesOrNetwork(),
"The files in this test are relatively small. In such cases AzCopy cannot tell whether performance was limited by "+
"your network, or by the additional processing overheads associated with small files. To check, run another benchmark using "+
"files at least 32 MB in size. That will provide a good test of your network speed, unaffected by file size. Then compare that speed "+
"to the speed measured just now, with the small files. If throughput is lower in the small-file case, that means file size "+
"is affecting performance. (AzCopy shows this message when files are less than or equal to %d MiB in size, and no other performance "+
"constraints were identified). In this test, throughput of %d Mega bits/sec was obtained with %d concurrent connections.",
htbbThresholdMB, p.mbps, p.finalConcurrency)
} else {
addAdvice(EAdviceType.NetworkIsBottleneck(),
"No other factors were identified that are limiting performance, so the bottleneck is assumed to be available "+
"network bandwidth. The available network bandwidth is the portion of the installed bandwidth that "+
"is not already used by other traffic. Throughput of %d Mega bits/sec was obtained with %d concurrent connections.",
p.mbps, p.finalConcurrency)
}
}
case concurrencyReasonHighCpu:
addAdvice(EAdviceType.ConcurrencyHighCpu(),
"When auto-tuning concurrency, AzCopy experienced high CPU usage so "+
"AzCopy might not have reached the full capacity of your network. Consider trying a machine or VM with more CPU power. "+
"(This is an experimental feature so, if you believe AzCopy was mistaken and CPU usage was actually fine, you can turn "+
"off this message by setting the environment variable %s to false.)", common.EEnvironmentVariable.AutoTuneToCpu().Name)
// TODO: review whether we still need the environment variable, and adjust this message if we remove it
case ConcurrencyReasonTunerDisabled:
addAdvice(EAdviceType.ConcurrencyNotTuned(),
"Auto-tuning of concurrency was prevented by an environment variable setting a specific concurrency value. Therefore "+
"AzCopy cannot tune itself to find the maximum possible throughput.")
case concurrencyReasonHitMax:
addAdvice(EAdviceType.ConcurrencyHitUpperLimit(),
"Auto-tuning of concurrency hit its upper limit before finding maximum throughput. Therefore the maximum "+
"possible throughput was not found")
default:
addAdvice(EAdviceType.ConcurrencyNotEnoughTime(),
"The job completed before AzCopy could find the maximum possible throughput. Try benchmarking with more files "+
"to give AzCopy more time. To do so, use the --%s command line parameter. Its default is value %d, so try "+
"benchmarking with it set to at least %d", common.FileCountParam, common.FileCountDefault,
common.FileCountDefault*20) // * 20 because default is calibrated for 1 Gbps, and 20 Gbps is a common max storage account speed
}
}
// TODO: consider how to factor in CPU load - will it be reflected in concurrency tuner results, or separate?
// TODO: should we also output aka.ms links to the relevant doc pages? Hard to maintain?
// TODO: do we need to add a case for files that are too small for HTBB (need to tell difference between premium block block
// blob and normal). Or will 503s tell us enough in those cases?
return result
}