in Solutions/Mimecast/Data Connectors/MimecastAudit/MimecastAudit/mimecast_audit_to_sentinel.py [0:0]
def get_mimecast_audit_data_in_sentinel(self):
"""Fetch the audit data and push into sentinel.
Raises:
MimecastException: MimecastException
MimecastException: KeyError
MimecastException: Unknown Exception
"""
__method_name = inspect.currentframe().f_code.co_name
try:
applogger.info(
self.log_format.format(
consts.LOGS_STARTS_WITH,
__method_name,
self.azure_function_name,
"Fetching and Pushing Audit Data from Mimecast",
)
)
checkpoint = self.checkpoint_field()
start_date = checkpoint.get("start_time")
end_date = checkpoint.get("end_time")
payload = self.set_payload(start_date, end_date)
if "next" in checkpoint and checkpoint["next"] != "":
token = checkpoint.get("next")
payload = self.set_payload(start_date, end_date, token=token)
applogger.info(
self.log_format.format(
consts.LOGS_STARTS_WITH,
__method_name,
self.azure_function_name,
"Continuing data ingestion from remaining ,from : {}".format(
start_date
),
)
)
has_more_data = True
while has_more_data:
has_more_data = self.valid_run()
if not has_more_data:
break
start_datetime = payload["data"][0]["startDateTime"]
end_datetime = payload["data"][0]["endDateTime"]
page_token = payload["meta"]["pagination"].get("pageToken", "")
applogger.info(
self.log_format.format(
consts.LOGS_STARTS_WITH,
__method_name,
self.azure_function_name,
"Making Mimecast API request with start date : {} end date : {} , token : {}".format(
start_datetime, end_datetime, page_token
),
)
)
response = self.make_rest_call(
method="POST",
url=consts.BASE_URL + consts.ENDPOINTS["AUDIT_ENDPOINT"],
json=payload,
)
data = response.get("data")
if len(data) > 0:
data_to_post = json.dumps(data)
sentinel.post_data(data_to_post, consts.TABLE_NAME["Audit"])
applogger.info(
self.log_format.format(
consts.LOGS_STARTS_WITH,
__method_name,
self.azure_function_name,
"Data ingested to Sentinel ,start date : {} end date : {} , count : {} ".format(
start_date, end_date, len(data)
),
)
)
else:
applogger.info(
self.log_format.format(
consts.LOGS_STARTS_WITH,
__method_name,
self.azure_function_name,
"No Data found",
)
)
if "next" in response["meta"]["pagination"]:
token = response["meta"]["pagination"]["next"]
checkpoint["next"] = token
checkpoint["start_time"] = start_datetime
checkpoint["end_time"] = end_datetime
self.post_checkpoint_data(
self.state_manager, checkpoint, dump_flag=True
)
payload = self.set_payload(start_date, end_date, token=token)
applogger.info(
self.log_format.format(
consts.LOGS_STARTS_WITH,
__method_name,
self.azure_function_name,
"Next token updated in checkpoint file",
)
)
else:
applogger.info(
self.log_format.format(
consts.LOGS_STARTS_WITH,
__method_name,
self.azure_function_name,
"Successfully completed the execution between start date :{} and end date : {}".format(
start_date, end_date
),
)
)
start_date, end_date = self.update_date_in_checkpoint()
payload = self.set_payload(start_date, end_date)
except MimecastException:
raise MimecastException()
except MimecastTimeoutException:
return
except KeyError as key_error:
applogger.error(
self.log_format.format(
consts.LOGS_STARTS_WITH,
__method_name,
self.azure_function_name,
consts.KEY_ERROR_MSG.format(key_error),
)
)
raise MimecastException()
except RetryError as error:
applogger.error(
self.log_format.format(
consts.LOGS_STARTS_WITH,
__method_name,
self.azure_function_name,
consts.MAX_RETRY_ERROR_MSG.format(
error, error.last_attempt.exception()
),
)
)
raise MimecastException()
except Exception as err:
applogger.error(
self.log_format.format(
consts.LOGS_STARTS_WITH,
__method_name,
self.azure_function_name,
consts.UNEXPECTED_ERROR_MSG.format(err),
)
)
raise MimecastException()