in source/log_parser/log-parser.py [0:0]
def update_ip_set(log, ip_set_type, outstanding_requesters):
log.info('[update_ip_set] Start')
# With wafv2 api we need to pass the scope, name and arn of an IPSet to manipulate the Address list
# We also can only put source_ips in the appropriate IPSets based on IP version
# Depending on the ip_set_type, we choose the appropriate set of IPSets and Names
# initialize as SCANNER_PROBES IPSets
ipset_name_v4 = None
ipset_name_v6 = None
ipset_arn_v4 = None
ipset_arn_v6 = None
# switch if type of IPSets are HTTP_FLOOD
if ip_set_type == flood:
ipset_name_v4 = os.getenv('IP_SET_NAME_HTTP_FLOODV4')
ipset_name_v6 = os.getenv('IP_SET_NAME_HTTP_FLOODV6')
ipset_arn_v4 = os.getenv('IP_SET_ID_HTTP_FLOODV4')
ipset_arn_v6 = os.getenv('IP_SET_ID_HTTP_FLOODV6')
if ip_set_type == scanners:
ipset_name_v4 = os.getenv('IP_SET_NAME_SCANNERS_PROBESV4')
ipset_name_v6 = os.getenv('IP_SET_NAME_SCANNERS_PROBESV6')
ipset_arn_v4 = os.getenv('IP_SET_ID_SCANNERS_PROBESV4')
ipset_arn_v6 = os.getenv('IP_SET_ID_SCANNERS_PROBESV6')
counter = 0
try:
if ipset_arn_v4 == None or ipset_arn_v6 == None:
log.info("[update_ip_set] Ignore process when ip_set_id is None")
return
# --------------------------------------------------------------------------------------------------------------
log.info("[update_ip_set] \tMerge general and uriList into a single list")
# --------------------------------------------------------------------------------------------------------------
unified_outstanding_requesters = outstanding_requesters['general']
for uri in outstanding_requesters['uriList'].keys():
for k in outstanding_requesters['uriList'][uri].keys():
if (k not in unified_outstanding_requesters.keys() or
outstanding_requesters['uriList'][uri][k]['max_counter_per_min'] >
unified_outstanding_requesters[k]['max_counter_per_min']):
unified_outstanding_requesters[k] = outstanding_requesters['uriList'][uri][k]
# --------------------------------------------------------------------------------------------------------------
log.info("[update_ip_set] \tTruncate [if necessary] list to respect WAF limit")
# --------------------------------------------------------------------------------------------------------------
if len(unified_outstanding_requesters) > int(os.getenv('LIMIT_IP_ADDRESS_RANGES_PER_IP_MATCH_CONDITION')):
ordered_unified_outstanding_requesters = sorted(unified_outstanding_requesters.items(),
key=lambda kv: kv[1]['max_counter_per_min'], reverse=True)
unified_outstanding_requesters = {}
for key, value in ordered_unified_outstanding_requesters:
if counter < int(os.getenv('LIMIT_IP_ADDRESS_RANGES_PER_IP_MATCH_CONDITION')):
unified_outstanding_requesters[key] = value
counter += 1
else:
break
# --------------------------------------------------------------------------------------------------------------
log.info("[update_ip_set] \tBlock remaining outstanding requesters")
# --------------------------------------------------------------------------------------------------------------
addresses_v4 = []
addresses_v6 = []
for k in unified_outstanding_requesters.keys():
ip_type = waflib.which_ip_version(log, k)
source_ip = waflib.set_ip_cidr(log, k)
if ip_type == "IPV4":
addresses_v4.append(source_ip)
elif ip_type == "IPV6":
addresses_v6.append(source_ip)
# --------------------------------------------------------------------------------------------------------------
log.info("[update_ip_set] \tCommit changes in WAF IP set")
# --------------------------------------------------------------------------------------------------------------
response = waflib.update_ip_set(log, scope, ipset_name_v4, ipset_arn_v4, addresses_v4)
# Sleep for a few seconds to mitigate AWS WAF Update API call throttling issue
sleep(delay_between_updates)
response = waflib.update_ip_set(log, scope, ipset_name_v6, ipset_arn_v6, addresses_v6)
except Exception as error:
log.error(str(error))
log.error("[update_ip_set] Error to update waf ip set")
log.info('[update_ip_set] End')
return counter