in cmd/promtool/unittest.go [167:371]
func (tg *testGroup) test(mint, maxt time.Time, evalInterval time.Duration, groupOrderMap map[string]int, ruleFiles ...string) []error {
// Setup testing suite.
suite, err := promql.NewLazyLoader(nil, tg.seriesLoadingString())
if err != nil {
return []error{err}
}
defer suite.Close()
// Load the rule files.
opts := &rules.ManagerOptions{
QueryFunc: rules.EngineQueryFunc(suite.QueryEngine(), suite.Storage()),
Appendable: suite.Storage(),
Context: context.Background(),
NotifyFunc: func(ctx context.Context, expr string, alerts ...*rules.Alert) {},
Logger: log.NewNopLogger(),
}
m := rules.NewManager(opts)
groupsMap, ers := m.LoadGroups(tg.Interval, tg.ExternalLabels, ruleFiles...)
if ers != nil {
return ers
}
groups := orderedGroups(groupsMap, groupOrderMap)
// Pre-processing some data for testing alerts.
// All this preparation is so that we can test alerts as we evaluate the rules.
// This avoids storing them in memory, as the number of evals might be high.
// All the `eval_time` for which we have unit tests for alerts.
alertEvalTimesMap := map[time.Duration]struct{}{}
// Map of all the eval_time+alertname combination present in the unit tests.
alertsInTest := make(map[time.Duration]map[string]struct{})
// Map of all the unit tests for given eval_time.
alertTests := make(map[time.Duration][]alertTestCase)
for _, alert := range tg.AlertRuleTests {
alertEvalTimesMap[alert.EvalTime] = struct{}{}
if _, ok := alertsInTest[alert.EvalTime]; !ok {
alertsInTest[alert.EvalTime] = make(map[string]struct{})
}
alertsInTest[alert.EvalTime][alert.Alertname] = struct{}{}
alertTests[alert.EvalTime] = append(alertTests[alert.EvalTime], alert)
}
alertEvalTimes := make([]time.Duration, 0, len(alertEvalTimesMap))
for k := range alertEvalTimesMap {
alertEvalTimes = append(alertEvalTimes, k)
}
sort.Slice(alertEvalTimes, func(i, j int) bool {
return alertEvalTimes[i] < alertEvalTimes[j]
})
// Current index in alertEvalTimes what we are looking at.
curr := 0
var errs []error
for ts := mint; ts.Before(maxt); ts = ts.Add(evalInterval) {
// Collects the alerts asked for unit testing.
suite.WithSamplesTill(ts, func(err error) {
if err != nil {
errs = append(errs, err)
return
}
for _, g := range groups {
g.Eval(suite.Context(), ts)
for _, r := range g.Rules() {
if r.LastError() != nil {
errs = append(errs, errors.Errorf(" rule: %s, time: %s, err: %v",
r.Name(), ts.Sub(time.Unix(0, 0)), r.LastError()))
}
}
}
})
if len(errs) > 0 {
return errs
}
for {
if !(curr < len(alertEvalTimes) && ts.Sub(mint) <= alertEvalTimes[curr] &&
alertEvalTimes[curr] < ts.Add(evalInterval).Sub(mint)) {
break
}
// We need to check alerts for this time.
// If 'ts <= `eval_time=alertEvalTimes[curr]` < ts+evalInterval'
// then we compare alerts with the Eval at `ts`.
t := alertEvalTimes[curr]
presentAlerts := alertsInTest[t]
got := make(map[string]labelsAndAnnotations)
// Same Alert name can be present in multiple groups.
// Hence we collect them all to check against expected alerts.
for _, g := range groups {
grules := g.Rules()
for _, r := range grules {
ar, ok := r.(*rules.AlertingRule)
if !ok {
continue
}
if _, ok := presentAlerts[ar.Name()]; !ok {
continue
}
var alerts labelsAndAnnotations
for _, a := range ar.ActiveAlerts() {
if a.State == rules.StateFiring {
alerts = append(alerts, labelAndAnnotation{
Labels: append(labels.Labels{}, a.Labels...),
Annotations: append(labels.Labels{}, a.Annotations...),
})
}
}
got[ar.Name()] = append(got[ar.Name()], alerts...)
}
}
for _, testcase := range alertTests[t] {
// Checking alerts.
gotAlerts := got[testcase.Alertname]
var expAlerts labelsAndAnnotations
for _, a := range testcase.ExpAlerts {
// User gives only the labels from alerting rule, which doesn't
// include this label (added by Prometheus during Eval).
if a.ExpLabels == nil {
a.ExpLabels = make(map[string]string)
}
a.ExpLabels[labels.AlertName] = testcase.Alertname
expAlerts = append(expAlerts, labelAndAnnotation{
Labels: labels.FromMap(a.ExpLabels),
Annotations: labels.FromMap(a.ExpAnnotations),
})
}
if gotAlerts.Len() != expAlerts.Len() {
errs = append(errs, errors.Errorf(" alertname:%s, time:%s, \n exp:%#v, \n got:%#v",
testcase.Alertname, testcase.EvalTime.String(), expAlerts.String(), gotAlerts.String()))
} else {
sort.Sort(gotAlerts)
sort.Sort(expAlerts)
if !reflect.DeepEqual(expAlerts, gotAlerts) {
errs = append(errs, errors.Errorf(" alertname:%s, time:%s, \n exp:%#v, \n got:%#v",
testcase.Alertname, testcase.EvalTime.String(), expAlerts.String(), gotAlerts.String()))
}
}
}
curr++
}
}
// Checking promql expressions.
Outer:
for _, testCase := range tg.PromqlExprTests {
got, err := query(suite.Context(), testCase.Expr, mint.Add(testCase.EvalTime),
suite.QueryEngine(), suite.Queryable())
if err != nil {
errs = append(errs, errors.Errorf(" expr: %q, time: %s, err: %s", testCase.Expr,
testCase.EvalTime.String(), err.Error()))
continue
}
var gotSamples []parsedSample
for _, s := range got {
gotSamples = append(gotSamples, parsedSample{
Labels: s.Metric.Copy(),
Value: s.V,
})
}
var expSamples []parsedSample
for _, s := range testCase.ExpSamples {
lb, err := promql.ParseMetric(s.Labels)
if err != nil {
err = errors.Wrapf(err, "labels %q", s.Labels)
errs = append(errs, errors.Errorf(" expr: %q, time: %s, err: %s", testCase.Expr,
testCase.EvalTime.String(), err.Error()))
continue Outer
}
expSamples = append(expSamples, parsedSample{
Labels: lb,
Value: s.Value,
})
}
sort.Slice(expSamples, func(i, j int) bool {
return labels.Compare(expSamples[i].Labels, expSamples[j].Labels) <= 0
})
sort.Slice(gotSamples, func(i, j int) bool {
return labels.Compare(gotSamples[i].Labels, gotSamples[j].Labels) <= 0
})
if !reflect.DeepEqual(expSamples, gotSamples) {
errs = append(errs, errors.Errorf(" expr: %q, time: %s,\n exp:%#v\n got:%#v", testCase.Expr,
testCase.EvalTime.String(), parsedSamplesString(expSamples), parsedSamplesString(gotSamples)))
}
}
if len(errs) > 0 {
return errs
}
return nil
}