in eks/configmaps/remote/configmaps.go [758:1045]
func (ts *tester) checkResults() (err error) {
curTS := time.Now().UTC().Format(time.RFC3339Nano)
ts.cfg.Logger.Info("checking results", zap.String("timestamp", curTS))
writesSummary := metrics.RequestsSummary{TestID: curTS}
curWriteLatencies := make(metrics.Durations, 0, 20000)
writesDirRaw := ""
writesDirSummary := ""
writesDirRaw, err = aws_s3.DownloadDir(
ts.cfg.Logger,
ts.cfg.S3API,
ts.cfg.EKSConfig.S3.BucketName,
path.Dir(ts.cfg.EKSConfig.AddOnConfigmapsRemote.RequestsRawWritesJSONS3Key),
)
if err == nil {
ts.cfg.Logger.Info("reading writes results raw",
zap.String("writes-dir", writesDirRaw),
zap.String("s3-dir", path.Dir(ts.cfg.EKSConfig.AddOnConfigmapsRemote.RequestsRawWritesJSONS3Key)),
)
cnt := 0
err = filepath.Walk(writesDirRaw, func(fpath string, info os.FileInfo, werr error) error {
if werr != nil {
return werr
}
if info.IsDir() {
return nil
}
cnt++
switch {
case strings.HasSuffix(fpath, "-writes-raw.json"):
b, err := ioutil.ReadFile(fpath)
if err != nil {
return fmt.Errorf("failed to open %q (%v)", fpath, err)
}
var r metrics.Durations
if err = json.Unmarshal(b, &r); err != nil {
return fmt.Errorf("failed to unmarshal %q (%s, %v)", fpath, string(b), err)
}
curWriteLatencies = append(curWriteLatencies, r...)
}
return nil
})
if err != nil || cnt == 0 {
ts.cfg.Logger.Warn("failed to read writes results", zap.Int("file-count", cnt), zap.Error(err))
os.RemoveAll(writesDirRaw)
writesDirRaw = ""
}
}
writesDirSummary, err = aws_s3.DownloadDir(
ts.cfg.Logger,
ts.cfg.S3API,
ts.cfg.EKSConfig.S3.BucketName,
path.Dir(ts.cfg.EKSConfig.AddOnConfigmapsRemote.RequestsSummaryWritesJSONS3Key),
)
if err == nil {
ts.cfg.Logger.Info("reading writes results summary",
zap.String("writes-dir", writesDirSummary),
zap.String("s3-dir", path.Dir(ts.cfg.EKSConfig.AddOnConfigmapsRemote.RequestsSummaryWritesJSONS3Key)),
)
cnt := 0
err = filepath.Walk(writesDirSummary, func(fpath string, info os.FileInfo, werr error) error {
if werr != nil {
return werr
}
if info.IsDir() {
return nil
}
cnt++
switch {
case strings.HasSuffix(fpath, "-writes-summary.json"):
b, err := ioutil.ReadFile(fpath)
if err != nil {
return fmt.Errorf("failed to open %q (%v)", fpath, err)
}
var r metrics.RequestsSummary
if err = json.Unmarshal(b, &r); err != nil {
return fmt.Errorf("failed to unmarshal %q (%s, %v)", fpath, string(b), err)
}
writesSummary.SuccessTotal += r.SuccessTotal
writesSummary.FailureTotal += r.FailureTotal
if writesSummary.LatencyHistogram == nil || len(writesSummary.LatencyHistogram) == 0 {
writesSummary.LatencyHistogram = r.LatencyHistogram
} else {
writesSummary.LatencyHistogram, err = metrics.MergeHistograms(writesSummary.LatencyHistogram, r.LatencyHistogram)
if err != nil {
return fmt.Errorf("failed to merge histograms (%v)", err)
}
}
}
return nil
})
if err != nil || cnt == 0 {
ts.cfg.Logger.Warn("failed to read writes results", zap.Int("file-count", cnt), zap.Error(err))
os.RemoveAll(writesDirSummary)
writesDirSummary = ""
}
}
sortStart := time.Now()
ts.cfg.Logger.Info("sorting write latencies", zap.Int("data", len(curWriteLatencies)))
sort.Sort(curWriteLatencies)
ts.cfg.Logger.Info("sorted write latencies", zap.String("took", time.Since(sortStart).String()))
writesSummary.LantencyP50 = curWriteLatencies.PickLantencyP50()
writesSummary.LantencyP90 = curWriteLatencies.PickLantencyP90()
writesSummary.LantencyP99 = curWriteLatencies.PickLantencyP99()
writesSummary.LantencyP999 = curWriteLatencies.PickLantencyP999()
writesSummary.LantencyP9999 = curWriteLatencies.PickLantencyP9999()
ts.cfg.EKSConfig.AddOnConfigmapsRemote.RequestsSummaryWrites = writesSummary
ts.cfg.EKSConfig.Sync()
wb, err := json.Marshal(curWriteLatencies)
if err != nil {
ts.cfg.Logger.Warn("failed to encode JSON", zap.Error(err))
return err
}
if err = ioutil.WriteFile(ts.cfg.EKSConfig.AddOnConfigmapsRemote.RequestsRawWritesJSONPath, wb, 0600); err != nil {
ts.cfg.Logger.Warn("failed to write file", zap.Error(err))
return err
}
if err = aws_s3.Upload(
ts.cfg.Logger,
ts.cfg.S3API,
ts.cfg.EKSConfig.S3.BucketName,
ts.cfg.EKSConfig.AddOnConfigmapsRemote.RequestsRawWritesJSONS3Key,
ts.cfg.EKSConfig.AddOnConfigmapsRemote.RequestsRawWritesJSONPath,
); err != nil {
return err
}
if err = ioutil.WriteFile(ts.cfg.EKSConfig.AddOnConfigmapsRemote.RequestsSummaryWritesJSONPath, []byte(writesSummary.JSON()), 0600); err != nil {
ts.cfg.Logger.Warn("failed to write file", zap.Error(err))
return err
}
if err = aws_s3.Upload(
ts.cfg.Logger,
ts.cfg.S3API,
ts.cfg.EKSConfig.S3.BucketName,
ts.cfg.EKSConfig.AddOnConfigmapsRemote.RequestsSummaryWritesJSONS3Key,
ts.cfg.EKSConfig.AddOnConfigmapsRemote.RequestsSummaryWritesJSONPath,
); err != nil {
return err
}
if err = ioutil.WriteFile(ts.cfg.EKSConfig.AddOnConfigmapsRemote.RequestsSummaryWritesTablePath, []byte(writesSummary.Table()), 0600); err != nil {
ts.cfg.Logger.Warn("failed to write file", zap.Error(err))
return err
}
if err = aws_s3.Upload(
ts.cfg.Logger,
ts.cfg.S3API,
ts.cfg.EKSConfig.S3.BucketName,
ts.cfg.EKSConfig.AddOnConfigmapsRemote.RequestsSummaryWritesTableS3Key,
ts.cfg.EKSConfig.AddOnConfigmapsRemote.RequestsSummaryWritesTablePath,
); err != nil {
return err
}
fmt.Fprintf(ts.cfg.LogWriter, "\n\nRequestsSummaryWrites:\n%s\n", writesSummary.Table())
s3Objects := make([]*s3.Object, 0)
if ts.cfg.EKSConfig.AddOnConfigmapsRemote.RequestsSummaryWritesCompareS3Dir != "" {
s3Objects, err = aws_s3.ListInDescendingLastModified(
ts.cfg.Logger,
ts.cfg.S3API,
ts.cfg.EKSConfig.S3.BucketName,
path.Clean(ts.cfg.EKSConfig.AddOnConfigmapsRemote.RequestsSummaryWritesCompareS3Dir)+"/",
)
}
canCompare := len(s3Objects) > 0 && err == nil
if canCompare {
reqSummaryS3Key := aws.StringValue(s3Objects[0].Key)
durRawS3Key := path.Join(ts.cfg.EKSConfig.AddOnConfigmapsRemote.RequestsRawWritesCompareS3Dir, path.Base(reqSummaryS3Key))
var prevSummary metrics.RequestsSummary
prevSummary, err = metrics.DownloadRequestsSummaryFromS3(ts.cfg.Logger, ts.cfg.S3API, ts.cfg.EKSConfig.S3.BucketName, reqSummaryS3Key)
if err != nil {
ts.cfg.Logger.Warn("failed to download results", zap.Error(err))
return err
}
ts.cfg.EKSConfig.AddOnConfigmapsRemote.RequestsSummaryWritesCompare, err = metrics.CompareRequestsSummary(prevSummary, ts.cfg.EKSConfig.AddOnConfigmapsRemote.RequestsSummaryWrites)
if err != nil {
ts.cfg.Logger.Warn("failed to compare results", zap.Error(err))
return err
}
if err = ioutil.WriteFile(ts.cfg.EKSConfig.AddOnConfigmapsRemote.RequestsSummaryWritesCompareJSONPath, []byte(ts.cfg.EKSConfig.AddOnConfigmapsRemote.RequestsSummaryWritesCompare.JSON()), 0600); err != nil {
ts.cfg.Logger.Warn("failed to write file", zap.Error(err))
return err
}
if err = aws_s3.Upload(
ts.cfg.Logger,
ts.cfg.S3API,
ts.cfg.EKSConfig.S3.BucketName,
ts.cfg.EKSConfig.AddOnConfigmapsRemote.RequestsSummaryWritesCompareJSONS3Key,
ts.cfg.EKSConfig.AddOnConfigmapsRemote.RequestsSummaryWritesCompareJSONPath,
); err != nil {
return err
}
if err = ioutil.WriteFile(ts.cfg.EKSConfig.AddOnConfigmapsRemote.RequestsSummaryWritesCompareTablePath, []byte(ts.cfg.EKSConfig.AddOnConfigmapsRemote.RequestsSummaryWritesCompare.Table()), 0600); err != nil {
ts.cfg.Logger.Warn("failed to write file", zap.Error(err))
return err
}
if err = aws_s3.Upload(
ts.cfg.Logger,
ts.cfg.S3API,
ts.cfg.EKSConfig.S3.BucketName,
ts.cfg.EKSConfig.AddOnConfigmapsRemote.RequestsSummaryWritesCompareTableS3Key,
ts.cfg.EKSConfig.AddOnConfigmapsRemote.RequestsSummaryWritesCompareTablePath,
); err != nil {
return err
}
fmt.Fprintf(ts.cfg.LogWriter, "\n\nRequestsSummaryWritesCompare:\n%s\n", ts.cfg.EKSConfig.AddOnConfigmapsRemote.RequestsSummaryWritesCompare.Table())
var prevDurations metrics.Durations
prevDurations, err = metrics.DownloadDurationsFromS3(ts.cfg.Logger, ts.cfg.S3API, ts.cfg.EKSConfig.S3.BucketName, durRawS3Key)
if err != nil {
ts.cfg.Logger.Warn("failed to download results", zap.Error(err))
return err
}
prevDurationsWithLabels := metrics.LabelDurations(prevDurations, prevSummary.TestID)
curDurationsWithLabels := metrics.LabelDurations(curWriteLatencies, ts.cfg.EKSConfig.AddOnConfigmapsRemote.RequestsSummaryWrites.TestID)
allDurationsWithLabels := append(prevDurationsWithLabels, curDurationsWithLabels...)
sortStart := time.Now()
ts.cfg.Logger.Info("sorting before and after durations with label",
zap.Int("before-data-points", len(prevDurationsWithLabels)),
zap.Int("after-data-points", len(curDurationsWithLabels)),
zap.Int("total-points", len(allDurationsWithLabels)),
)
sort.Sort(allDurationsWithLabels)
ts.cfg.Logger.Info("sorted before and after durations with label",
zap.Int("before-data-points", len(prevDurationsWithLabels)),
zap.Int("after-data-points", len(curDurationsWithLabels)),
zap.Int("total-points", len(allDurationsWithLabels)),
zap.String("took", time.Since(sortStart).String()),
)
allDataJSON, err := json.Marshal(allDurationsWithLabels)
if err != nil {
ts.cfg.Logger.Warn("failed to marshal results", zap.Error(err))
return err
}
if err = ioutil.WriteFile(ts.cfg.EKSConfig.AddOnConfigmapsRemote.RequestsRawWritesCompareAllJSONPath, []byte(allDataJSON), 0600); err != nil {
ts.cfg.Logger.Warn("failed to write file", zap.Error(err))
return err
}
if err = aws_s3.Upload(
ts.cfg.Logger,
ts.cfg.S3API,
ts.cfg.EKSConfig.S3.BucketName,
ts.cfg.EKSConfig.AddOnConfigmapsRemote.RequestsRawWritesCompareAllJSONS3Key,
ts.cfg.EKSConfig.AddOnConfigmapsRemote.RequestsRawWritesCompareAllJSONPath,
); err != nil {
return err
}
if err = allDurationsWithLabels.CSV(ts.cfg.EKSConfig.AddOnConfigmapsRemote.RequestsRawWritesCompareAllCSVPath); err != nil {
return err
}
if err = aws_s3.Upload(
ts.cfg.Logger,
ts.cfg.S3API,
ts.cfg.EKSConfig.S3.BucketName,
ts.cfg.EKSConfig.AddOnConfigmapsRemote.RequestsRawWritesCompareAllCSVS3Key,
ts.cfg.EKSConfig.AddOnConfigmapsRemote.RequestsRawWritesCompareAllCSVPath,
); err != nil {
return err
}
} else {
ts.cfg.Logger.Warn("previous writes summary not found; skipping comparison", zap.Error(err))
}
ts.cfg.Logger.Info("uploading new writes summary to s3 bucket to overwrite the previous")
if err = aws_s3.Upload(
ts.cfg.Logger,
ts.cfg.S3API,
ts.cfg.EKSConfig.S3.BucketName,
path.Join(ts.cfg.EKSConfig.AddOnConfigmapsRemote.RequestsRawWritesCompareS3Dir, curTS),
ts.cfg.EKSConfig.AddOnConfigmapsRemote.RequestsRawWritesJSONPath,
); err != nil {
return err
}
if err = aws_s3.Upload(
ts.cfg.Logger,
ts.cfg.S3API,
ts.cfg.EKSConfig.S3.BucketName,
path.Join(ts.cfg.EKSConfig.AddOnConfigmapsRemote.RequestsSummaryWritesCompareS3Dir, curTS),
ts.cfg.EKSConfig.AddOnConfigmapsRemote.RequestsSummaryWritesJSONPath,
); err != nil {
return err
}
ts.cfg.EKSConfig.Sync()
return nil
}