in Solutions/VMware SD-WAN and SASE/Data Connectors/Function App Connector/vmw_sdwan_sase_funcapp/sdwan_efslogs/__init__.py [0:0]
def veco_fwlog_delayadjust():
logging.info("FUNCTION-EFSDELAY: Starting Search API delay measurement.")
# Re-adjust delay information in state so that the next iteration is aware of logging issues.
# Return value structure defined:
j_epoch_delay = {
"data": {
"1hr": {
"event_count": 0,
"avg_delay": 0,
"measurement_unit": "msec"
},
"4hr": {
"event_count": 0,
"avg_delay": 0,
"measurement_unit": "msec"
},
"8hr": {
"event_count": 0,
"avg_delay": 0,
"measurement_unit": "msec"
}
},
"metadata": {
"loganalytics_api_result": 0,
"new_state_stored": False,
"veco_searchapi_result": 000
}
}
# Run 0 to 0 query (from=0&size=0) to see how many events are in the log storage from the past 1/4/8 hours, we will use this as a baseline for calculating delay spread
query_end = datetime.datetime.utcnow().isoformat()
query_start = {
"now-1hrs": datetime.datetime.isoformat(datetime.datetime.fromisoformat(query_end) - datetime.timedelta(hours=1)) + "Z",
"now-4hrs": datetime.datetime.isoformat(datetime.datetime.fromisoformat(query_end) - datetime.timedelta(hours=4)) + "Z",
"now-8hrs": datetime.datetime.isoformat(datetime.datetime.fromisoformat(query_end) - datetime.timedelta(hours=8)) + "Z"
}
# Again, for verbosity, capture delays (now-timestamp)
j_delays = {
"1hr": [],
"4hr": [],
"8hr": []
}
for query_start_timestamp in query_start:
params= "/edgeFirewall?from=0&size=0&startTime=" + str(query_start[query_start_timestamp]) + "&endTime=" + str(query_end) + "Z"
query = craftAPIurl(j_config_list["host"], "/api/search/v1/enterprises/", j_config_list["token"], True, params)
logging.info("FUNCTION-EFSDELAY: API Call to: " + query)
header = {
"Authorization": j_config_list["token"]
}
nullrequest = requests.get(url=query, headers=header)
if nullrequest.status_code != 200:
# If the API call fails, skip next steps
logging.error("FUNCTION-EFSDELAY: Unexpected error when sending API call, terminating function...")
j_epoch_delay["metadata"]["veco_searchapi_result"] = nullrequest.status_code
return j_epoch_delay
else:
j_count_response = nullrequest.json()
logging.warning("Firewall events: " + str(j_count_response["count"]))
date_format = "%Y-%m-%dT%H:%M:%S.000Z"
if j_count_response["count"] != 0:
# 200 OK and events were found, go in and extract them
if query_start[query_start_timestamp] == query_start["now-1hrs"]:
# Get the first 100 events form the past hour to calculate average delay
params_1hr = "/edgeFirewall?from=0&size=100&startTime=" + str(query_start[query_start_timestamp]) + "&endTime=" + str(query_end) + "Z"
query = craftAPIurl(j_config_list["host"], "/api/search/v1/enterprises/", j_config_list["token"], True, params_1hr)
past1hrrequest = requests.get(url=query, headers=header)
if past1hrrequest.status_code != 200:
# If the API call fails, skip next steps
logging.error("FUNCTION-EFSDELAY: Unexpected error when sending API call, terminating function...")
j_epoch_delay["metadata"]["veco_searchapi_result"] = nullrequest.status_code
return j_epoch_delay
else:
j_log_items = past1hrrequest.json()
logging.info("FUNCTION-EFSDELAY: Found firewall events to process in the past 1 hours: " + str(j_log_items["count"]))
j_epoch_delay["data"]["1hr"]["event_count"] = j_log_items["count"]
for log_item in j_log_items["data"]:
delay = datetime.datetime.fromisoformat(query_end) - datetime.datetime.strptime(log_item["_source"]["timestamp"], date_format)
j_delays["1hr"].append(int(delay.total_seconds() * 1000))
if query_start[query_start_timestamp] == query_start["now-4hrs"]:
# Get the first 100 events form the past 4 hours to calculate average delay
params_4hr = "/edgeFirewall?from=0&size=100&startTime=" + str(query_start[query_start_timestamp]) + "&endTime=" + str(query_end) + "Z"
query = craftAPIurl(j_config_list["host"], "/api/search/v1/enterprises/", j_config_list["token"], True, params_4hr)
past4hrrequest = requests.get(url=query, headers=header)
if past4hrrequest.status_code != 200:
# If the API call fails, skip next steps
logging.error("FUNCTION-EFSDELAY: Unexpected error when sending API call, terminating function...")
j_epoch_delay["metadata"]["veco_searchapi_result"] = nullrequest.status_code
return j_epoch_delay
else:
j_log_items = past4hrrequest.json()
logging.info("FUNCTION-EFSDELAY: Found firewall events to process in the past 4 hours: " + str(j_log_items["count"]))
j_epoch_delay["data"]["4hr"]["event_count"] = j_log_items["count"]
for log_item in j_log_items["data"]:
delay = datetime.datetime.fromisoformat(query_end) - datetime.datetime.strptime(log_item["_source"]["timestamp"], date_format)
j_delays["4hr"].append(int(delay.total_seconds() * 1000))
if query_start[query_start_timestamp] == query_start["now-8hrs"]:
logging.info("FUNCTION-EFSDELAY: Found firewall logs to process in the past 8 hours: " + str(j_count_response["count"]))
# Get the first 100 events form the past 8 hours to calculate average delay
params_8hr = "/edgeFirewall?from=0&size=100&startTime=" + str(query_start[query_start_timestamp]) + "&endTime=" + str(query_end) + "Z"
query = craftAPIurl(j_config_list["host"], "/api/search/v1/enterprises/", j_config_list["token"], True, params_8hr)
past8hrrequest = requests.get(url=query, headers=header)
if past8hrrequest.status_code != 200:
# If the API call fails, skip next steps
logging.error("FUNCTION-EFSDELAY: Unexpected error when sending API call, terminating function...")
j_epoch_delay["metadata"]["veco_searchapi_result"] = nullrequest.status_code
return j_epoch_delay
else:
j_log_items = past8hrrequest.json()
logging.info("FUNCTION-EFSDELAY: Found firewall events to process in the past 8 hours: " + str(j_log_items["count"]))
j_epoch_delay["data"]["8hr"]["event_count"] = j_log_items["count"]
for log_item in j_log_items["data"]:
delay = datetime.datetime.fromisoformat(query_end) - datetime.datetime.strptime(log_item["_source"]["timestamp"], date_format)
j_delays["8hr"].append(int(delay.total_seconds() * 1000))
# Average calculated delays in milliseconds
# FIXME: sanitize this portion, very pedestrian code, but verbosity is advisable
avg_count = 3
sum = 0
count = 0
if j_delays["1hr"] != []:
for delay_item in j_delays["1hr"]:
sum = sum + delay_item
count = count + 1
avgdelay_1hr = sum // count
j_epoch_delay["data"]["1hr"]["avg_delay"] = avgdelay_1hr
else:
avg_count = avg_count - 1
avgdelay_1hr = 0
if j_delays["4hr"] != []:
for delay_item in j_delays["4hr"]:
sum = sum + delay_item
count = count + 1
avgdelay_4hr = sum // count
j_epoch_delay["data"]["4hr"]["avg_delay"] = avgdelay_4hr
else:
avg_count = avg_count - 1
avgdelay_4hr = 0
if j_delays["8hr"] != []:
for delay_item in j_delays["8hr"]:
sum = sum + delay_item
count = count + 1
avgdelay_8hr = sum // count
j_epoch_delay["data"]["8hr"]["avg_delay"] = avgdelay_8hr
else:
avgdelay_8hr = 0
logging.warning("FUNCTION-EFSDELAY: The script measured the first 100 events.")
avgdelay = (avgdelay_1hr + avgdelay_4hr + avgdelay_8hr) // avg_count
logging.info("FUNCTION-EFSDELAY: 1 hours measured average delay is: " + str(avgdelay_1hr) + "msec, 4 hours: " + str(avgdelay_4hr) + "msecs, 8 hours: " + str(avgdelay_8hr) + "msec")
logging.info("FUNCTION-EFSDELAY: Next iteration will use the delay of : " + str(avgdelay) + " msecs, this is cca. " + str(avgdelay / 1000 / 60 / 60) + " hours.")
# Now that we have the delay measured, we will write the state data to the storage account.
logging.info("FUNCTION-EFSDELAY: Measurement complete, writing to file share...")
statefile = ShareFileClient.from_connection_string(conn_str=os.environ["azsa_share_connectionstring"], share_name=os.environ["azsa_share_name"], file_path="function_state/state.json")
g_state["services"]["efs"]["delay_value"] = avgdelay
g_state["services"]["efs"]["delay_unit"] = "msec"
g_state["services"]["efs"]["update_timestamp"]: str(datetime.datetime.utcnow())
statefile.upload_file(data=json.dumps(g_state))
statefile.close()
j_epoch_delay["metadata"]["new_state_stored"] = True
j_epoch_delay["metadata"]["veco_searchapi_result"] = 200
if j_count_response["count"] == 0:
logging.warning("FUNCTION-EFSDELAY: The delay measured by the script might be larger than 8 hours. If you suspect that you should be seeing firewall logs from the past 8 hours, please raise a case on my.vmware.com.")
callLogAnalyticsAPI(j_epoch_delay, j_config_list["logingestion_api"]["dce"], j_config_list["logingestion_api"]["sdwan"]["efs_health"]["imi"], j_config_list["logingestion_api"]["sdwan"]["efs_health"]["stream"])
return j_epoch_delay