in google-datacatalog-apache-atlas-connector/src/google/datacatalog_connectors/apache_atlas/scrape/metadata_event_scraper.py [0:0]
def __scrape_entity_events(self, entity_events=None):
entity_events, event_consumer = self.__retrieve_entity_events(
entity_events)
if entity_events:
logging.info('Scraping Entity Events...')
self._log_scrape_start(
'Scraping Metadata for {} entity events...'.format(
len(entity_events)))
classifications_dict = {}
entity_types_dict = {}
enum_types_dict = {}
self._log_scrape_start('Scraping admin metrics...')
admin_metrics = self._apache_atlas_facade.get_admin_metrics()
logging.info(admin_metrics)
self._log_single_object_scrape_result(admin_metrics)
self._log_scrape_start('Scraping typedefs...')
for typedef in self._apache_atlas_facade.get_typedefs():
self._scrape_classification_types(classifications_dict,
typedef)
self._scrape_enum_types(enum_types_dict, typedef)
self.__scrape_entity_types_for_events(entity_events,
entity_types_dict,
typedef)
metadata_enricher.MetadataEnricher.enrich_entity_relationships(
entity_types_dict)
return {
'classifications': classifications_dict,
'enum_types': enum_types_dict,
'entity_types': entity_types_dict,
'entity_events': entity_events
}, event_consumer
else:
logging.info('No Entity Events...')
return {
'classifications': {},
'enum_types': {},
'entity_types': {},
'entity_events': []
}, event_consumer