in source/log_parser/log-parser.py [0:0]
def send_anonymous_usage_data(log):
try:
if 'SEND_ANONYMOUS_USAGE_DATA' not in environ or os.getenv('SEND_ANONYMOUS_USAGE_DATA').lower() != 'yes':
return
log.info("[send_anonymous_usage_data] Start")
cw = create_client('cloudwatch')
usage_data = {
"data_type": "log_parser",
"scanners_probes_set_size": 0,
"http_flood_set_size": 0,
"allowed_requests": 0,
"blocked_requests_all": 0,
"blocked_requests_scanners_probes": 0,
"blocked_requests_http_flood": 0,
"allowed_requests_WAFWebACL": 0,
"blocked_requests_WAFWebACL": 0,
"waf_type": os.getenv('LOG_TYPE')
}
# --------------------------------------------------------------------------------------------------------------
log.info("[send_anonymous_usage_data] Get num allowed requests")
# --------------------------------------------------------------------------------------------------------------
try:
response = cw.get_metric_statistics(
MetricName='AllowedRequests',
Namespace='AWS/WAFV2',
Statistics=['Sum'],
Period=300,
StartTime=datetime.datetime.utcnow() - datetime.timedelta(seconds=300),
EndTime=datetime.datetime.utcnow(),
Dimensions=[
{
"Name": "Rule",
"Value": "ALL"
},
{
"Name": "WebACL",
"Value": os.getenv('STACK_NAME')
},
{
"Name": "Region",
"Value": os.getenv('AWS_REGION')
}
]
)
if len(response['Datapoints']):
usage_data['allowed_requests'] = response['Datapoints'][0]['Sum']
except Exception as error:
log.debug("[send_anonymous_usage_data] Failed to get Num Allowed Requests")
log.debug(str(error))
# --------------------------------------------------------------------------------------------------------------
log.info("[send_anonymous_usage_data] Get num blocked requests - all rules")
# --------------------------------------------------------------------------------------------------------------
try:
response = cw.get_metric_statistics(
MetricName='BlockedRequests',
Namespace='AWS/WAFV2',
Statistics=['Sum'],
Period=300,
StartTime=datetime.datetime.utcnow() - datetime.timedelta(seconds=300),
EndTime=datetime.datetime.utcnow(),
Dimensions=[
{
"Name": "Rule",
"Value": "ALL"
},
{
"Name": "WebACL",
"Value": os.getenv('STACK_NAME')
},
{
"Name": "Region",
"Value": os.getenv('AWS_REGION')
}
]
)
if len(response['Datapoints']):
usage_data['blocked_requests_all'] = response['Datapoints'][0]['Sum']
except Exception as error:
log.info("[send_anonymous_usage_data] Failed to get num blocked requests - all rules")
log.error(str(error))
# --------------------------------------------------------------------------------------------------------------
log.info("[send_anonymous_usage_data] Get scanners probes data")
# --------------------------------------------------------------------------------------------------------------
if 'IP_SET_ID_SCANNERS_PROBESV4' in environ or 'IP_SET_ID_SCANNERS_PROBESV6' in environ:
try:
countv4 = 0
response = waflib.get_ip_set(log, scope,
os.getenv('IP_SET_NAME_SCANNERS_PROBESV4'),
os.getenv('IP_SET_ID_SCANNERS_PROBESV4')
)
log.info(response)
if response is not None:
countv4 = len(response['IPSet']['Addresses'])
log.info("Scanner Probes IPV4 address Count: %s", countv4)
countv6 = 0
response = waflib.get_ip_set(log, scope,
os.getenv('IP_SET_NAME_SCANNERS_PROBESV6'),
os.getenv('IP_SET_ID_SCANNERS_PROBESV6')
)
log.info(response)
if response is not None:
countv6 = len(response['IPSet']['Addresses'])
log.info("Scanner Probes IPV6 address Count: %s", countv6)
usage_data['scanners_probes_set_size'] = str(countv4 + countv6)
response = cw.get_metric_statistics(
MetricName='BlockedRequests',
Namespace='AWS/WAFV2',
Statistics=['Sum'],
Period=300,
StartTime=datetime.datetime.utcnow() - datetime.timedelta(seconds=300),
EndTime=datetime.datetime.utcnow(),
Dimensions=[
{
"Name": "Rule",
"Value": os.getenv('METRIC_NAME_PREFIX') + 'ScannersProbesRule'
},
{
"Name": "WebACL",
"Value": os.getenv('STACK_NAME')
},
{
"Name": "Region",
"Value": os.getenv('AWS_REGION')
}
]
)
if len(response['Datapoints']):
usage_data['blocked_requests_scanners_probes'] = response['Datapoints'][0]['Sum']
except Exception as error:
log.debug("[send_anonymous_usage_data] Failed to get scanners probes data")
log.debug(str(error))
# --------------------------------------------------------------------------------------------------------------
log.info("[send_anonymous_usage_data] Get HTTP flood data")
# --------------------------------------------------------------------------------------------------------------
if 'IP_SET_ID_HTTP_FLOODV4' in environ or 'IP_SET_ID_HTTP_FLOODV6' in environ:
try:
countv4 = 0
response = waflib.get_ip_set(log, scope,
os.getenv('IP_SET_NAME_HTTP_FLOODV4'),
os.getenv('IP_SET_ID_HTTP_FLOODV4')
)
log.info(response)
if response is not None:
countv4 = len(response['IPSet']['Addresses'])
log.info("HTTP Flood IPV4 address Count: %s", countv4)
countv6 = 0
response = waflib.get_ip_set(log, scope,
os.getenv('IP_SET_NAME_HTTP_FLOODV6'),
os.getenv('IP_SET_ID_HTTP_FLOODV6')
)
log.info(response)
if response is not None:
countv6 = len(response['IPSet']['Addresses'])
log.info("HTTP Flood IPV6 address Count: %s", countv6)
usage_data['http_flood_set_size'] = str(countv4 + countv6)
response = cw.get_metric_statistics(
MetricName='BlockedRequests',
Namespace='AWS/WAFV2',
Statistics=['Sum'],
Period=300,
StartTime=datetime.datetime.utcnow() - datetime.timedelta(seconds=300),
EndTime=datetime.datetime.utcnow(),
Dimensions=[
{
"Name": "Rule",
"Value": os.getenv('METRIC_NAME_PREFIX') + 'HttpFloodRegularRule'
},
{
"Name": "WebACL",
"Value": os.getenv('STACK_NAME')
},
{
"Name": "Region",
"Value": os.getenv('AWS_REGION')
}
]
)
if len(response['Datapoints']):
usage_data['blocked_requests_http_flood'] = response['Datapoints'][0]['Sum']
except Exception as error:
log.info("[send_anonymous_usage_data] Failed to get HTTP flood data")
log.error(str(error))
# --------------------------------------------------------------------------------------------------------------
log.info("[send_anonymous_usage_data] Get num allowed requests - WAF Web ACL")
# --------------------------------------------------------------------------------------------------------------
try:
response = cw.get_metric_statistics(
MetricName='AllowedRequests',
Namespace='AWS/WAFV2',
Statistics=['Sum'],
Period=300,
StartTime=datetime.datetime.utcnow() - datetime.timedelta(seconds=300),
EndTime=datetime.datetime.utcnow(),
Dimensions=[
{
"Name": "Rule",
"Value": os.getenv('METRIC_NAME_PREFIX') + 'WAFWebACL'
},
{
"Name": "WebACL",
"Value": os.getenv('STACK_NAME')
},
{
"Name": "Region",
"Value": os.getenv('AWS_REGION')
}
]
)
if len(response['Datapoints']):
usage_data['allowed_requests_WAFWebACL'] = response['Datapoints'][0]['Sum']
except Exception as error:
log.info("[send_anonymous_usage_data] Failed to get num blocked requests - all rules")
log.error(str(error))
# --------------------------------------------------------------------------------------------------------------
log.info("[send_anonymous_usage_data] Get num blocked requests - WAF Web ACL")
# --------------------------------------------------------------------------------------------------------------
try:
response = cw.get_metric_statistics(
MetricName='BlockedRequests',
Namespace='AWS/WAFV2',
Statistics=['Sum'],
Period=300,
StartTime=datetime.datetime.utcnow() - datetime.timedelta(seconds=300),
EndTime=datetime.datetime.utcnow(),
Dimensions=[
{
"Name": "Rule",
"Value": os.getenv('METRIC_NAME_PREFIX') + 'WAFWebACL'
},
{
"Name": "WebACL",
"Value": os.getenv('STACK_NAME')
},
{
"Name": "Region",
"Value": os.getenv('AWS_REGION')
}
]
)
if len(response['Datapoints']):
usage_data['blocked_requests_WAFWebACL'] = response['Datapoints'][0]['Sum']
except Exception as error:
log.info("[send_anonymous_usage_data] Failed to get num blocked requests - all rules")
log.error(str(error))
# --------------------------------------------------------------------------------------------------------------
log.info("[send_anonymous_usage_data] Send Data")
# --------------------------------------------------------------------------------------------------------------
response = send_metrics(data=usage_data)
response_code = response.status_code
log.info('[send_anonymous_usage_data] Response Code: {}'.format(response_code))
log.info("[send_anonymous_usage_data] End")
except Exception as error:
log.info("[send_anonymous_usage_data] Failed to send data")
log.error(str(error))