in Solutions/CiscoUmbrella/Data Connectors/ciscoUmbrellaDataConn/__init__.py [0:0]
def main(mytimer: func.TimerRequest) -> None:
if mytimer.past_due:
logging.info('The timer is past due!')
logging.info('Starting program')
script_start_time = int(time.time())
state_manager_cu = StateManager(FILE_SHARE_CONN_STRING, file_path='cisco_umbrella')
ts_from = state_manager_cu.get()
ts_to = datetime.datetime.utcnow() - datetime.timedelta(minutes=1)
if ts_from is not None:
if (datetime.datetime.utcnow() - datetime.timedelta(days=3)) > datetime.datetime.strptime(ts_from,"%Y-%m-%dT%H:%M:%S.%fZ"):
ts_from = parse_date_from(ts_from)
ts_to = ts_from + datetime.timedelta(days=1)
else:
ts_to = datetime.datetime.utcnow() - datetime.timedelta(minutes=1)
ts_from = parse_date_from(ts_from)
ts_to = ts_to.replace(tzinfo=datetime.timezone.utc, second=0, microsecond=0)
cli = UmbrellaClient(aws_access_key_id, aws_secret_acces_key, aws_s3_bucket)
logging.info('Searching files last modified from {} to {}'.format(ts_from, ts_to))
obj_list = cli.get_files_list(ts_from, ts_to)
logging.info('Total number of files is {}. Total size is {} MB'.format(
len(obj_list),
round(sum([x['Size'] for x in obj_list]) / 10**6, 2)
))
failed_sent_events_number = 0
successfull_sent_events_number = 0
if DIVIDE_TO_MULTIPLE_TABLES:
sentinel_dict = {
'dns': AzureSentinelConnector(logAnalyticsUri, sentinel_customer_id, sentinel_shared_key, sentinel_log_type + '_dns', queue_size=10000, bulks_number=10),
'proxy': AzureSentinelConnector(logAnalyticsUri, sentinel_customer_id, sentinel_shared_key, sentinel_log_type + '_proxy', queue_size=10000, bulks_number=10),
'ip': AzureSentinelConnector(logAnalyticsUri, sentinel_customer_id, sentinel_shared_key, sentinel_log_type + '_ip', queue_size=10000, bulks_number=10),
'cloudfirewall': AzureSentinelConnector(logAnalyticsUri, sentinel_customer_id, sentinel_shared_key, sentinel_log_type + '_cloudfirewall', queue_size=10000, bulks_number=10),
'firewall': AzureSentinelConnector(logAnalyticsUri, sentinel_customer_id, sentinel_shared_key, sentinel_log_type + '_firewall', queue_size=10000, bulks_number=10)
}
last_ts = None
for obj in sorted(obj_list, key=lambda k: k['LastModified']):
key = obj.get('Key', '')
if 'dnslogs' in key.lower():
sentinel = sentinel_dict['dns']
elif 'proxylogs' in key.lower():
sentinel = sentinel_dict['proxy']
elif 'iplogs' in key.lower():
sentinel = sentinel_dict['ip']
elif 'cloudfirewalllogs' in key.lower() or 'cdfwlogs' in key.lower():
sentinel = sentinel_dict['cloudfirewall']
elif 'firewalllogs' in key.lower():
sentinel = sentinel_dict['firewall']
else:
# skip files of unknown types
continue
with sentinel:
cli.process_file(obj, dest=sentinel)
last_ts = obj['LastModified']
if last_ts:
state_manager_cu.post(datetime.datetime.strftime(last_ts, '%Y-%m-%dT%H:%M:%S.%fZ'))
if check_if_script_runs_too_long(script_start_time):
logging.info(f'Script is running too long. Stop processing new events. Finish script.')
break
else:
state_manager_cu.post(datetime.datetime.strftime(ts_to, '%Y-%m-%dT%H:%M:%S.%fZ'))
if check_if_script_runs_too_long(script_start_time):
logging.info(f'Script is running too long. Stop processing new events. Finish script.')
break
if last_ts:
state_manager_cu.post(datetime.datetime.strftime(last_ts, '%Y-%m-%dT%H:%M:%S.%fZ'))
else:
state_manager_cu.post(datetime.datetime.strftime(ts_to, '%Y-%m-%dT%H:%M:%S.%fZ'))
failed_sent_events_number = sum([sentinel.failed_sent_events_number for sentinel in sentinel_dict.values()])
successfull_sent_events_number = sum([sentinel.successfull_sent_events_number for sentinel in sentinel_dict.values()])
else:
sentinel = AzureSentinelConnector(logAnalyticsUri, sentinel_customer_id, sentinel_shared_key, sentinel_log_type, queue_size=10000, bulks_number=10)
with sentinel:
for obj in sorted(obj_list, key=lambda k: k['LastModified']):
cli.process_file(obj, dest=sentinel)
last_ts = obj['LastModified']
if last_ts:
state_manager_cu.post(datetime.datetime.strftime(last_ts, '%Y-%m-%dT%H:%M:%S.%fZ'))
if check_if_script_runs_too_long(script_start_time):
logging.info(f'Script is running too long. Stop processing new events. Finish script.')
return
else:
state_manager_cu.post(datetime.datetime.strftime(ts_to, '%Y-%m-%dT%H:%M:%S.%fZ'))
if check_if_script_runs_too_long(script_start_time):
logging.info(f'Script is running too long. Stop processing new events. Finish script.')
return
if last_ts:
state_manager_cu.post(datetime.datetime.strftime(last_ts, '%Y-%m-%dT%H:%M:%S.%fZ'))
else:
state_manager_cu.post(datetime.datetime.strftime(ts_to, '%Y-%m-%dT%H:%M:%S.%fZ'))
failed_sent_events_number += sentinel.failed_sent_events_number
successfull_sent_events_number += sentinel.successfull_sent_events_number
if failed_sent_events_number:
logging.error('{} events have not been sent'.format(failed_sent_events_number))
logging.info('Program finished. {} events have been sent. {} events have not been sent'.format(successfull_sent_events_number, failed_sent_events_number))