in gloo/benchmark/runner.cc [540:591]
void Runner::checkErrors() {
// Only check if that option has been set
if (!options_.verify) {
return;
}
// If there were no mismatches, don't print anything
if (mismatchErrors_.empty()) {
return;
}
// If there were mismatches, print them
int size = mismatchErrors_.size();
// Add barrier to prevent header from printing before benchmark results
barrier_->run();
printVerifyHeader();
if (options_.contextRank == 0) {
// Only print this stuff once
if (size >= kMaxNumErrors && !options_.showAllErrors) {
// If too many errors, inform user we will only be printing a subset
std::cout << "Too many errors! Truncating to top 20 from each rank. ";
std::cout << std::endl;
std::cout << "Use the flag --show-all-errors to see all errors.";
std::cout << std::endl << std::endl;
}
}
if (size >= kMaxNumErrors && !options_.showAllErrors) {
// Truncate errors if too many and user did not want to see all
size = 20;
}
// Prints the errors from each rank in order
// Since each rank is run on a different process, we occasionally have higher
// ranks beginning to output their errors before lower ones causing overlaps.
// This is confusing for the user so use a loop and a barrier at the beginning
// of each iteration. This will force the processes to sync each time,
// thus the output will be printed in the correct order.
for (int i = 0; i < options_.contextSize; ++i) {
barrier_->run();
if (i != options_.contextRank) {
// Skip if it is not current rank's turn
continue;
}
for (int j = 0; j < size; ++j) {
std::cout << mismatchErrors_[j] << std::endl;
}
}
// Print footer and then exit program
barrier_->run();
printFooter();
// Exit with error
exit(1);
}