in x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go [137:293]
func (m *MetricSet) Fetch(report mb.ReporterV2) error {
// Get startTime and endTime
startTime, endTime := aws.GetStartTimeEndTime(time.Now(), m.Period, m.Latency, m.PreviousEndTime)
m.PreviousEndTime = endTime
m.Logger().Debugf("startTime = %s, endTime = %s", startTime, endTime)
// Initialise the map that will be used in case APIGateway api is configured. Infoapi includes Name_of_API:ID_of_API entries
infoapi := make(map[string]string)
// Check statistic method in config
err := m.checkStatistics()
if err != nil {
return fmt.Errorf("checkStatistics failed: %w", err)
}
// Get listMetricDetailTotal and namespaceDetailTotal from configuration
listMetricDetailTotal, namespaceDetailTotal := m.readCloudwatchConfig()
m.logger.Debugf("listMetricDetailTotal = %s", listMetricDetailTotal)
m.logger.Debugf("namespaceDetailTotal = %s", namespaceDetailTotal)
var config aws.Config
err = m.Module().UnpackConfig(&config)
if err != nil {
return err
}
// Starting from Go 1.24, when FIPS 140-3 mode is active, fips140.Enabled() will return true.
// So, regardless of whether `fips_enabled` is set to true or false, when FIPS 140-3 mode is active, the
// resolver will resolve to the FIPS endpoint.
// See: https://go.dev/doc/security/fips140#fips-140-3-mode
if fips140.Enabled() {
config.AWSConfig.FIPSEnabled = true
}
// Create events based on listMetricDetailTotal from configuration
if len(listMetricDetailTotal.metricsWithStats) != 0 {
for _, regionName := range m.MetricSet.RegionsList {
m.logger.Debugf("Collecting metrics from AWS region %s", regionName)
beatsConfig := m.MetricSet.AwsConfig.Copy()
beatsConfig.Region = regionName
APIClients, err := m.createAwsRequiredClients(beatsConfig, regionName, config)
if err != nil {
m.Logger().Warn("skipping metrics list from region '%s'", regionName)
}
eventsWithIdentifier, err := m.createEvents(APIClients.CloudWatchClient, APIClients.Resourcegroupstaggingapi, listMetricDetailTotal.metricsWithStats, listMetricDetailTotal.resourceTypeFilters, infoapi, regionName, startTime, endTime)
if err != nil {
return fmt.Errorf("createEvents failed for region %s: %w", regionName, err)
}
m.logger.Debugf("Collected metrics of metrics = %d", len(eventsWithIdentifier))
for _, event := range eventsWithIdentifier {
_ = event.RootFields.Delete(aws.CloudWatchPeriodName)
report.Event(event)
}
}
}
// Create events based on namespaceDetailTotal from configuration
for _, regionName := range m.MetricSet.RegionsList {
m.logger.Debugf("Collecting metrics from AWS region %s", regionName)
beatsConfig := m.MetricSet.AwsConfig.Copy()
beatsConfig.Region = regionName
APIClients, err := m.createAwsRequiredClients(beatsConfig, regionName, config)
if err != nil {
m.Logger().Warn("skipping metrics list from region '%s'", regionName, err)
continue
}
// retrieve all the details for all the metrics available in the current region when no namespace is specified
// otherwise only retrieve metrics from the specific namespaces from the config
var listMetricsOutput []aws.MetricWithID
if len(namespaceDetailTotal) == 0 {
listMetricsOutput, err = aws.GetListMetricsOutput("*", regionName, m.Period, m.IncludeLinkedAccounts, m.OwningAccount, m.MonitoringAccountID, APIClients.CloudWatchClient)
if err != nil {
m.Logger().Errorf("Error while retrieving the list of metrics for region %s and namespace %s: %w", regionName, "*", err)
}
} else {
for namespace := range namespaceDetailTotal {
listMetricsOutputPerNamespace, err := aws.GetListMetricsOutput(namespace, regionName, m.Period, m.IncludeLinkedAccounts, m.OwningAccount, m.MonitoringAccountID, APIClients.CloudWatchClient)
if err != nil {
m.Logger().Errorf("Error while retrieving the list of metrics for region %s and namespace %s: %w", regionName, namespace, err)
}
listMetricsOutput = append(listMetricsOutput, listMetricsOutputPerNamespace...)
}
}
if len(listMetricsOutput) == 0 {
continue
}
for namespace, namespaceDetails := range namespaceDetailTotal {
m.logger.Debugf("Collected metrics from namespace %s", namespace)
// filter listMetricsOutput by detailed configuration per each namespace
filteredMetricWithStatsTotal := filterListMetricsOutput(listMetricsOutput, namespace, namespaceDetails)
// get resource type filters and tags filters for each namespace
resourceTypeTagFilters := constructTagsFilters(namespaceDetails)
//Check whether namespace is APIGW
if strings.Contains(strings.ToLower(namespace), checkns_lower) {
useonlyrest := false
if len(resourceTypeTagFilters) == 1 {
for key := range resourceTypeTagFilters {
if strings.Compare(strings.ToLower(key), checkresource_type_lower) == 0 {
useonlyrest = true
}
}
}
// inforestapi includes only Rest APIs
if useonlyrest {
infoapi, err = aws.GetAPIGatewayRestAPIOutput(APIClients.Apigateway, config.LimitRestAPI)
if err != nil {
m.Logger().Errorf("could not get rest apis output: %v", err)
}
} else {
// infoapi includes only Rest APIs
// apiGatewayAPI includes only WebSocket and HTTP APIs
infoapi, err = aws.GetAPIGatewayRestAPIOutput(APIClients.Apigateway, config.LimitRestAPI)
if err != nil {
m.Logger().Errorf("could not get rest apis output: %v", err)
}
apiGatewayAPI, err := aws.GetAPIGatewayAPIOutput(APIClients.Apigatewayv2)
if err != nil {
m.Logger().Errorf("could not get http and websocket apis output: %v", err)
}
if len(apiGatewayAPI) > 0 {
maps.Copy(infoapi, apiGatewayAPI)
}
}
m.Logger().Debugf("infoapi response: %v", infoapi)
}
eventsWithIdentifier, err := m.createEvents(APIClients.CloudWatchClient, APIClients.Resourcegroupstaggingapi, filteredMetricWithStatsTotal, resourceTypeTagFilters, infoapi, regionName, startTime, endTime)
if err != nil {
return fmt.Errorf("createEvents failed for region %s: %w", regionName, err)
}
m.logger.Debugf("Collected number of metrics = %d", len(eventsWithIdentifier))
events, err := addMetadata(m.logger, namespace, regionName, beatsConfig, config.AWSConfig.FIPSEnabled, eventsWithIdentifier)
if err != nil {
// TODO What to do if add metadata fails? I guess to continue, probably we have an 90% of reliable data
m.Logger().Warnf("could not add metadata to events: %v", err)
}
for _, event := range events {
_ = event.RootFields.Delete(aws.CloudWatchPeriodName)
report.Event(event)
}
}
}
return nil
}