in Solutions/GreyNoiseThreatIntelligence/Data Connectors/GreyNoiseAPISentinelConnector/main.py [0:0]
def consume_ips(self):
logging.info(
"Starting consumption of GreyNoise indicators with query %s"
% (self.greynoise_query)
)
total_addresses = 0 # counter for total IPs consumed
payload_size = None # total IPs available from query
tries = int(self.greynoise_tries)
scroll = "" # scroll token for pagination
complete = False
# MS Graph TI Upload API limits are 100 indicators per request and 100 requests per minute.
# Get MSAL token
token, msal_expiry_time = self.get_token()
if token:
logging.info("MSAL token obtained")
while not complete:
if datetime.datetime.now() > msal_expiry_time:
logging.info("MSAL token expiring soon, getting new token")
token, msal_expiry_time = self.get_token()
if token:
logging.info("MSAL token obtained")
try:
if self.greynoise_size != 0 and self.greynoise_size<= 2000:
payload = self.session.query(
query=self.greynoise_query,
size=self.greynoise_size,
scroll=scroll,
exclude_raw=True,
)
else:
payload = self.session.query(
query=self.greynoise_query,
scroll=scroll,
size=2000,
exclude_raw=True,
)
# this protects from bad / invalid queries
# and exits out before proceeding
if payload["count"] == 0:
logging.info("GreyNoise Query return no results, exiting")
sys.exit(1)
# Capture the total number of indicators available
elif payload["count"] and payload_size is None:
payload_size = int(payload["count"])
logging.info("Total Indicators found: %s results" % payload_size)
# Loop to generate STIX objects and upload to Sentinel
stix_objects = []
counter = 0
chunk_size = 100 # MS Graph TI Upload API limits are 100 indicators per request and 100 requests per minute.
for batch in self.chunks(payload["data"], chunk_size):
for gn_object in batch:
stix_object = self.gn_stix_generator.generate_indicator(gn_object)
expected_chunk_size = len(batch)
stix_objects.append(stix_object)
counter += 1
if counter == expected_chunk_size:
# send batch to sentinel
self.upload_indicators_to_sentinel(token, stix_objects)
# reset counter and stix_objects
counter = 0
stix_objects = []
# logging.info("Sent 100 GreyNoise indicators to Sentinel" )
# the scroll is for pagination but does not always exist because
# we have consumed all the IPs
scroll = payload.get("scroll")
complete = payload["complete"]
addresses = len(payload["data"])
total_addresses += addresses
logging.info(
"Sent %s GreyNoise indicators to Sentinel for a total of %s addresses"
% (addresses, total_addresses)
)
# this is a hacky workaround to deal with an edge case on the API where if
# you limit the results on a query, the complete flag doesn't flip to
# true correctly
if (
self.greynoise_size == 0
# and self.greynoise_size < int(payload["count"]) # noqa: W503
and total_addresses >= payload_size # noqa: W503
):
break
elif (
self.greynoise_size != 0
and self.greynoise_size < int(payload["count"]) # noqa: W503
and self.greynoise_size <= total_addresses # noqa: W503
):
break
except Exception as reqErr:
logging.error("Uploading IPs failed: %s" % str(reqErr))
if tries != 0:
tries -= 1
logging.error("Trying again in 10 seconds using same scroll...")
time.sleep(10)
else:
logging.error(
"Exiting program. Max tries met. With time str%s and last scroll: %s"
% (str(time), scroll)
)
sys.exit(3)
logging.info(
"Ingest process completed. Inserted %s Indicators into Microsoft Sentinel Threat Intelligence."
% total_addresses
)