in pkg/updater/updater.go [1281:1379]
func alertRow(cols []*statepb.Column, row *statepb.Row, failuresToOpen, passesToClose int, useCommitAsBuildID bool, userPropertyName string, columnHeader []*configpb.TestGroup_ColumnHeader) *statepb.AlertInfo {
if failuresToOpen == 0 {
return nil
}
var concurrentFailures int
var totalFailures int32
var passes int
var compressedIdx int
f := result.Iter(row.Results)
var firstFail *statepb.Column
var latestFail *statepb.Column
var latestPass *statepb.Column
var failIdx int
var latestFailIdx int
customColumnHeaders := make(map[string]string)
// find the first number of consecutive passesToClose (no alert)
// or else failuresToOpen (alert).
for _, col := range cols {
// TODO(fejta): ignore old running
rawRes, _ := f()
res := result.Coalesce(rawRes, result.IgnoreRunning)
if res == statuspb.TestStatus_NO_RESULT {
if rawRes == statuspb.TestStatus_RUNNING {
compressedIdx++
}
continue
}
if res == statuspb.TestStatus_PASS {
passes++
if concurrentFailures >= failuresToOpen {
if latestPass == nil {
latestPass = col // most recent pass before outage
}
if passes >= passesToClose {
break // enough failures and enough passes, definitely past the start of the failure
}
} else if passes >= passesToClose {
return nil // enough passes but not enough failures, there is no outage
} else {
concurrentFailures = 0
}
}
if res == statuspb.TestStatus_FAIL {
passes = 0
latestPass = nil
concurrentFailures++
totalFailures++
if totalFailures == 1 { // note most recent failure for this outage
latestFailIdx = compressedIdx
latestFail = col
}
failIdx = compressedIdx
firstFail = col
}
if res == statuspb.TestStatus_FLAKY {
passes = 0
if concurrentFailures >= failuresToOpen {
break // cannot definitively say which commit is at fault
}
concurrentFailures = 0
}
compressedIdx++
for i := 0; i < len(columnHeader); i++ {
if i >= len(col.Extra) {
logrus.WithFields(logrus.Fields{
"started": time.Unix(0, int64(col.GetStarted()*float64(time.Millisecond))),
"additionalColumnHeaders": col.GetExtra(),
}).Trace("Insufficient column header values to record.")
break
}
if columnHeader[i].Label != "" {
customColumnHeaders[columnHeader[i].Label] = col.Extra[i]
} else if columnHeader[i].Property != "" {
customColumnHeaders[columnHeader[i].Property] = col.Extra[i]
} else {
customColumnHeaders[columnHeader[i].ConfigurationValue] = col.Extra[i]
}
}
}
if concurrentFailures < failuresToOpen {
return nil
}
var id string
var latestID string
if len(row.CellIds) > 0 { // not all rows have cell ids
id = row.CellIds[failIdx]
latestID = row.CellIds[latestFailIdx]
}
msg := row.Messages[latestFailIdx]
var userProperties map[string]string
if row.UserProperty != nil && latestFailIdx < len(row.UserProperty) && row.UserProperty[latestFailIdx] != "" {
userProperties = map[string]string{
userPropertyName: row.UserProperty[latestFailIdx],
}
}
return alertInfo(totalFailures, msg, id, latestID, userProperties, firstFail, latestFail, latestPass, useCommitAsBuildID, customColumnHeaders)
}