in ozhera-intelligence/ozhera-intelligence-service/src/main/java/org/apache/ozhera/intelligence/service/MetricsService.java [68:164]
public HeraRootCaseAnalyseRes queryMetricsRootAnalysis(MetricsQueryParam req) {
HeraRootCaseAnalyseRes res = new HeraRootCaseAnalyseRes();
try {
updateParam(req);
log.info("updated param is : " + req);
String kubePodInfoPromQl = getKubePodInfoPromQl(req.getIp());
log.info("getPodNameByKubePodInfo.kubePodInfoPromQl:{}", kubePodInfoPromQl);
MetricRangeResponseVector podPodIpListVector = requestPrometheusRangeV2(kubePodInfoPromQl,
req.getStartTime(), req.getEndTime(), req.getEnv());
List<MetricDataRangeSetVector> kubePodInfo = podPodIpListVector.getData().getResult();
MetricDataRangeSetVector kubePodInfoMetrics = kubePodInfo.get(kubePodInfo.size() - 1);
String podName = kubePodInfoMetrics.getMetric().getPod();
String cpuPromQl = "max(rate(container_cpu_user_seconds_total{system=\"mione\",application=\"" + req.getApplication() + "\", pod=\"" + podName + "\", image!=\"\", container =~ \"^(?:\\\\d+-0-\\\\d+|faas-sidecar)$\"}[30s]) * 100)";
String loadPromQl = "max(container_cpu_load_average_10s{system=\"mione\",application=\"" + req.getApplication() + "\", pod=\"" + podName + "\", image!=\"\", container =~ \"^(?:\\\\d+-0-\\\\d+|faas-sidecar)$\"} / 1000)";
String gcMaxTimes = "max(delta(jvm_gc_pause_seconds_count{serverIp=\"" + req.getIp() + "\", application=\"" + req.getApplication() + "\",containerName=\"main\"}[1m]))";
String memoryUsedRate = "avg(sum(jvm_memory_used_bytes{application=\"" + req.getApplication() + "\",area=\"heap\",ip=\"" + req.getIp() + "\",containerName=\"main\"}) / sum(jvm_memory_max_bytes{application=\"" + req.getApplication() + "\",area=\"heap\",ip=\"" + req.getIp() + "\",containerName=\"main\"})) * 100";
String maxGCDuration = "max(max_over_time(jvm_gc_pause_seconds_max{serverIp=~\"" + req.getIp() + "\", application=\"" + req.getApplication() + "\",containerName=\"main\"}[1m]))";
String cpuCount = "system_cpu_count{serverIp=~\"" + req.getIp() + "\", application=\"" + req.getApplication() + "\",containerName=~\".*\"}";
log.info("getPodNameByKubePodInfo.cpuPromQl:{}", cpuPromQl);
log.info("getPodNameByKubePodInfo.loadPromQl:{}", loadPromQl);
log.info("getPodNameByKubePodInfo.gcMaxTimes:{}", gcMaxTimes);
log.info("getPodNameByKubePodInfo.memoryUsedRate:{}", memoryUsedRate);
log.info("getPodNameByKubePodInfo.maxGCDuration:{}", maxGCDuration);
log.info("getPodNameByKubePodInfo.cpuCount:{}", cpuCount);
CompletableFuture<MetricRangeResponseVector> cpuFuture = CompletableFuture.supplyAsync(() -> requestPrometheusRangeV2(cpuPromQl,
req.getStartTime(), req.getEndTime(), req.getEnv()), HERA_SOLUTION_METRICS_POOL);
CompletableFuture<MetricRangeResponseVector> loadFuture = CompletableFuture.supplyAsync(() -> requestPrometheusRangeV2(loadPromQl,
req.getStartTime(), req.getEndTime(), req.getEnv()), HERA_SOLUTION_METRICS_POOL);
CompletableFuture<MetricRangeResponseVector> gcMaxTimesFuture = CompletableFuture.supplyAsync(() -> requestPrometheusRangeV2(gcMaxTimes,
req.getStartTime(), req.getEndTime(), req.getEnv()), HERA_SOLUTION_METRICS_POOL);
CompletableFuture<MetricRangeResponseVector> memoryFuture = CompletableFuture.supplyAsync(() -> requestPrometheusRangeV2(memoryUsedRate,
req.getStartTime(), req.getEndTime(), req.getEnv()), HERA_SOLUTION_METRICS_POOL);
CompletableFuture<MetricRangeResponseVector> maxGCFuture = CompletableFuture.supplyAsync(() -> requestPrometheusRangeV2(maxGCDuration,
req.getStartTime(), req.getEndTime(), req.getEnv()), HERA_SOLUTION_METRICS_POOL);
CompletableFuture<MetricRangeResponseVector> cpuCountFuture = CompletableFuture.supplyAsync(() -> requestPrometheusRangeV2(cpuCount,
req.getStartTime(), req.getEndTime(), req.getEnv()), HERA_SOLUTION_METRICS_POOL);
CompletableFuture<Void> allComplete = CompletableFuture.allOf(cpuFuture, loadFuture, gcMaxTimesFuture, memoryFuture, maxGCFuture);
allComplete.get(8000, TimeUnit.MILLISECONDS);
MetricRangeResponseVector cpuMetrics = cpuFuture.get();
if (cpuMetrics != null) {
log.info("getPodNameByKubePodInfo cpuMetrics : " + cpuMetrics);
List<MetricDataRangeSetVector> cpuResult = cpuMetrics.getData().getResult();
if (cpuResult != null && cpuResult.size() > 0) {
List<List<String>> values = cpuResult.get(0).getValues();
res.setMaxCpuUsage(getMaxValue(values));
}
}
MetricRangeResponseVector loadMetrics = loadFuture.get();
if (loadMetrics != null) {
log.info("getPodNameByKubePodInfo loadMetrics : " + loadMetrics);
List<MetricDataRangeSetVector> loadResult = loadMetrics.getData().getResult();
if (loadResult != null && loadResult.size() > 0) {
List<List<String>> values = loadResult.get(0).getValues();
res.setMaxLoad(getMaxValue(values));
}
}
MetricRangeResponseVector gcMaxTimesMetrics = gcMaxTimesFuture.get();
if (gcMaxTimesMetrics != null) {
log.info("getPodNameByKubePodInfo gcMaxTimesMetrics : " + gcMaxTimesMetrics);
List<MetricDataRangeSetVector> gcMaxTimesResult = gcMaxTimesMetrics.getData().getResult();
if (gcMaxTimesResult != null && gcMaxTimesResult.size() > 0) {
List<List<String>> values = gcMaxTimesResult.get(0).getValues();
res.setMaxSTWCost(getMaxValue(values));
}
}
MetricRangeResponseVector memoryMetrics = memoryFuture.get();
if (memoryMetrics != null) {
log.info("getPodNameByKubePodInfo memoryMetrics : " + memoryMetrics);
List<MetricDataRangeSetVector> memoryResult = memoryMetrics.getData().getResult();
if (memoryResult != null && memoryResult.size() > 0) {
List<List<String>> values = memoryResult.get(0).getValues();
res.setMaxJvmHeapUsage(getMaxValue(values));
}
}
MetricRangeResponseVector maxGCMetrics = maxGCFuture.get();
if (maxGCMetrics != null) {
log.info("getPodNameByKubePodInfo maxGCMetrics : " + maxGCMetrics);
List<MetricDataRangeSetVector> maxGCResult = maxGCMetrics.getData().getResult();
if (maxGCResult != null && maxGCResult.size() > 0) {
List<List<String>> values = maxGCResult.get(0).getValues();
res.setSTWCountOf1m(getMaxValue(values));
}
}
MetricRangeResponseVector cpuCountMetrics = cpuCountFuture.get();
if (cpuCountMetrics != null) {
log.info("getPodNameByKubePodInfo maxGCMetrics : " + cpuCountMetrics);
List<MetricDataRangeSetVector> cpuCountResult = cpuCountMetrics.getData().getResult();
if (cpuCountResult != null && cpuCountResult.size() > 0) {
List<List<String>> values = cpuCountResult.get(0).getValues();
res.setCpuCount(getMaxValue(values));
}
}
} catch (Exception e) {
log.error("get root cause analysis error , ", e);
}
return res;
}