data_extraction_transformation/scripts/extract-alerts.py [9:115]: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - ''' This function is used in the get_alert_summary_info function. Provided the JSON of alerts related to a specific alert, it gets their ID and make them into a pipe '|' separated string ''' def extract_alert_ids(alerts): alert_ids = "" if (len(alerts) > 0): alert_ids = "|" for i in alerts: alert_ids = alert_ids + str(i['id']) + '|' return alert_ids ''' This function is used for extracting timestamps with an exception handling as some entries have the %Y-%m-%dT%H:%M:%S.%f timestamp format and others (entries from 2019 and prior) have %Y-%m-%dT%H:%M:%Stimestamp format ''' def parse_timestamp(timestamp_str): formats_to_try = ["%Y-%m-%dT%H:%M:%S.%f", "%Y-%m-%dT%H:%M:%S"] for fmt in formats_to_try: try: return datetime.strptime(timestamp_str, fmt) except ValueError: pass ''' This function takes a JSON of a specific test measurment associated with a performance-related alert and extracts its attributes ''' def get_alert_info(json_test): test_characteristics = {'single_alert_id': json_test['id'], 'single_alert_status': json_test['status'], 'single_alert_profile_url': json_test['profile_url'], 'single_alert_prev_profile_url': json_test['prev_profile_url'], 'single_alert_is_regression': json_test['is_regression'], 'single_alert_prev_value': json_test['prev_value'], 'single_alert_new_value': json_test['new_value'], 'single_alert_t_value': json_test['t_value'], 'single_alert_amount_abs': json_test['amount_abs'], 'single_alert_amount_pct': json_test['amount_pct'], 'single_alert_summary_id': json_test['summary_id'], 'single_alert_related_summary_id': json_test['related_summary_id'], 'single_alert_manually_created': json_test['manually_created'], 'single_alert_classifier': json_test['classifier'], 'single_alert_starred': json_test['starred'], 'single_alert_classifier_email': json_test['classifier_email'], 'single_alert_noise_profile': json_test['noise_profile'] } if (json_test['series_signature'] != None): test_characteristics['signature_id'] = json_test['series_signature']['id'] test_characteristics['single_alert_series_signature_framework_id'] = json_test['series_signature']['framework_id'] test_characteristics['single_alert_series_signature_signature_hash'] = json_test['series_signature']['signature_hash'] test_characteristics['single_alert_series_signature_machine_platform'] = json_test['series_signature']['machine_platform'] test_characteristics['single_alert_series_signature_suite'] = json_test['series_signature']['suite'] test_characteristics['single_alert_series_signature_test'] = json_test['series_signature']['test'] test_characteristics['single_alert_series_signature_lower_is_better'] = json_test['series_signature']['lower_is_better'] test_characteristics['single_alert_series_signature_has_subtests'] = json_test['series_signature']['has_subtests'] test_characteristics['single_alert_series_signature_option_collection_hash'] = json_test['series_signature']['option_collection_hash'] test_characteristics['single_alert_series_signature_tags'] = append_strings(json_test['series_signature']['tags']) test_characteristics['single_alert_series_signature_extra_options'] = append_strings(json_test['series_signature']['extra_options']) test_characteristics['single_alert_series_signature_measurement_unit'] = json_test['series_signature']['measurement_unit'] test_characteristics['single_alert_series_signature_suite_public_name'] = json_test['series_signature']['suite_public_name'] test_characteristics['single_alert_series_signature_test_public_name'] = json_test['series_signature']['test_public_name'] if (json_test['prev_taskcluster_metadata'] != None and len(json_test['prev_taskcluster_metadata']) > 0): test_characteristics['single_alert_prev_taskcluster_metadata_task_id'] = json_test['prev_taskcluster_metadata']['task_id'] test_characteristics['single_alert_prev_taskcluster_metadata_retry_id'] = json_test['prev_taskcluster_metadata']['retry_id'] if (json_test['taskcluster_metadata'] != None and len(json_test['taskcluster_metadata']) > 0): test_characteristics['single_alert_taskcluster_metadata_task_id'] = json_test['taskcluster_metadata']['task_id'] test_characteristics['single_alert_taskcluster_metadata_retry_id'] = json_test['taskcluster_metadata']['retry_id'] if (json_test['backfill_record'] != None): test_characteristics['single_alert_backfill_record_context'] = json_test['backfill_record']['context'], test_characteristics['single_alert_backfill_record_status'] = json_test['backfill_record']['status'], test_characteristics['single_alert_backfill_record_total_actions_triggered'] = json_test['backfill_record']['total_actions_triggered'], test_characteristics['single_alert_backfill_record_total_backfills_failed'] = json_test['backfill_record']['total_backfills_failed'], test_characteristics['single_alert_backfill_record_total_backfills_successful'] = json_test['backfill_record']['total_backfills_successful'], test_characteristics['single_alert_backfill_record_total_backfills_in_progress'] = json_test['backfill_record']['total_backfills_in_progress'], return test_characteristics ''' This function takes a JSON of a specific performance-related alert and extracts its attributes ''' def get_alert_summary_info(json_alert): alert_characteristics = {'alert_summary_id': json_alert['id'], 'alert_summary_push_id': json_alert['push_id'], 'alert_summary_prev_push_id': json_alert['prev_push_id'], 'alert_summary_creation_timestamp': json_alert['created'], 'alert_summary_first_triaged': json_alert['first_triaged'], 'alert_summary_repository': json_alert['repository'], 'alert_summary_framework': json_alert['framework'], 'alert_summary_triage_due_date': json_alert['triage_due_date'], 'alert_summary_related_alerts': extract_alert_ids(json_alert['related_alerts']), 'alert_summary_status': json_alert['status'], 'alert_summary_bug_number': json_alert['bug_number'], 'alert_summary_bug_due_date': json_alert['bug_due_date'], 'alert_summary_bug_updated': json_alert['bug_updated'], 'alert_summary_issue_tracker': json_alert['issue_tracker'], 'alert_summary_notes': json_alert['notes'], 'alert_summary_revision': json_alert['revision'], 'push_timestamp': json_alert['push_timestamp'], 'alert_prev_push_revision': json_alert['prev_push_revision'], 'alert_summary_assignee_username': json_alert['assignee_username'], 'alert_summary_assignee_email': json_alert['assignee_email'], 'alert_summary_performance_tags': append_strings(json_alert['performance_tags']) } return alert_characteristics def parse_args(): - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - data_extraction_transformation/scripts/update-still-processing-alerts.py [8:113]: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - ''' This function is used in the get_alert_summary_info function. Provided the JSON of alerts related to a specific alert, it gets their ID and make them into a pipe '|' separated string ''' def extract_alert_ids(alerts): alert_ids = "" if (len(alerts) > 0): alert_ids = "|" for i in alerts: alert_ids = alert_ids + str(i['id']) + '|' return alert_ids ''' This function is used for extracting timestamps with an exception handling as some entries have the %Y-%m-%dT%H:%M:%S.%f timestamp format and others (entries from 2019 and prior) have %Y-%m-%dT%H:%M:%Stimestamp format ''' def parse_timestamp(timestamp_str): formats_to_try = ["%Y-%m-%dT%H:%M:%S.%f", "%Y-%m-%dT%H:%M:%S"] for fmt in formats_to_try: try: return datetime.strptime(timestamp_str, fmt) except ValueError: pass ''' This function takes a JSON of a specific test measurment associated with a performance-related alert and extracts its attributes ''' def get_alert_info(json_test): test_characteristics = {'single_alert_id': json_test['id'], 'single_alert_status': json_test['status'], 'single_alert_profile_url': json_test['profile_url'], 'single_alert_prev_profile_url': json_test['prev_profile_url'], 'single_alert_is_regression': json_test['is_regression'], 'single_alert_prev_value': json_test['prev_value'], 'single_alert_new_value': json_test['new_value'], 'single_alert_t_value': json_test['t_value'], 'single_alert_amount_abs': json_test['amount_abs'], 'single_alert_amount_pct': json_test['amount_pct'], 'single_alert_summary_id': json_test['summary_id'], 'single_alert_related_summary_id': json_test['related_summary_id'], 'single_alert_manually_created': json_test['manually_created'], 'single_alert_classifier': json_test['classifier'], 'single_alert_starred': json_test['starred'], 'single_alert_classifier_email': json_test['classifier_email'], 'single_alert_noise_profile': json_test['noise_profile'] } if (json_test['series_signature'] != None): test_characteristics['signature_id'] = json_test['series_signature']['id'] test_characteristics['single_alert_series_signature_framework_id'] = json_test['series_signature']['framework_id'] test_characteristics['single_alert_series_signature_signature_hash'] = json_test['series_signature']['signature_hash'] test_characteristics['single_alert_series_signature_machine_platform'] = json_test['series_signature']['machine_platform'] test_characteristics['single_alert_series_signature_suite'] = json_test['series_signature']['suite'] test_characteristics['single_alert_series_signature_test'] = json_test['series_signature']['test'] test_characteristics['single_alert_series_signature_lower_is_better'] = json_test['series_signature']['lower_is_better'] test_characteristics['single_alert_series_signature_has_subtests'] = json_test['series_signature']['has_subtests'] test_characteristics['single_alert_series_signature_option_collection_hash'] = json_test['series_signature']['option_collection_hash'] test_characteristics['single_alert_series_signature_tags'] = append_strings(json_test['series_signature']['tags']) test_characteristics['single_alert_series_signature_extra_options'] = append_strings(json_test['series_signature']['extra_options']) test_characteristics['single_alert_series_signature_measurement_unit'] = json_test['series_signature']['measurement_unit'] test_characteristics['single_alert_series_signature_suite_public_name'] = json_test['series_signature']['suite_public_name'] test_characteristics['single_alert_series_signature_test_public_name'] = json_test['series_signature']['test_public_name'] if (json_test['prev_taskcluster_metadata'] != None and len(json_test['prev_taskcluster_metadata']) > 0): test_characteristics['single_alert_prev_taskcluster_metadata_task_id'] = json_test['prev_taskcluster_metadata']['task_id'] test_characteristics['single_alert_prev_taskcluster_metadata_retry_id'] = json_test['prev_taskcluster_metadata']['retry_id'] if (json_test['taskcluster_metadata'] != None and len(json_test['taskcluster_metadata']) > 0): test_characteristics['single_alert_taskcluster_metadata_task_id'] = json_test['taskcluster_metadata']['task_id'] test_characteristics['single_alert_taskcluster_metadata_retry_id'] = json_test['taskcluster_metadata']['retry_id'] if (json_test['backfill_record'] != None): test_characteristics['single_alert_backfill_record_context'] = json_test['backfill_record']['context'], test_characteristics['single_alert_backfill_record_status'] = json_test['backfill_record']['status'], test_characteristics['single_alert_backfill_record_total_actions_triggered'] = json_test['backfill_record']['total_actions_triggered'], test_characteristics['single_alert_backfill_record_total_backfills_failed'] = json_test['backfill_record']['total_backfills_failed'], test_characteristics['single_alert_backfill_record_total_backfills_successful'] = json_test['backfill_record']['total_backfills_successful'], test_characteristics['single_alert_backfill_record_total_backfills_in_progress'] = json_test['backfill_record']['total_backfills_in_progress'], return test_characteristics ''' This function takes a JSON of a specific performance-related alert and extracts its attributes ''' def get_alert_summary_info(json_alert): alert_characteristics = {'alert_summary_id': json_alert['id'], 'alert_summary_push_id': json_alert['push_id'], 'alert_summary_prev_push_id': json_alert['prev_push_id'], 'alert_summary_creation_timestamp': json_alert['created'], 'alert_summary_first_triaged': json_alert['first_triaged'], 'alert_summary_repository': json_alert['repository'], 'alert_summary_framework': json_alert['framework'], 'alert_summary_triage_due_date': json_alert['triage_due_date'], 'alert_summary_related_alerts': extract_alert_ids(json_alert['related_alerts']), 'alert_summary_status': json_alert['status'], 'alert_summary_bug_number': json_alert['bug_number'], 'alert_summary_bug_due_date': json_alert['bug_due_date'], 'alert_summary_bug_updated': json_alert['bug_updated'], 'alert_summary_issue_tracker': json_alert['issue_tracker'], 'alert_summary_notes': json_alert['notes'], 'alert_summary_revision': json_alert['revision'], 'push_timestamp': json_alert['push_timestamp'], 'alert_prev_push_revision': json_alert['prev_push_revision'], 'alert_summary_assignee_username': json_alert['assignee_username'], 'alert_summary_assignee_email': json_alert['assignee_email'], 'alert_summary_performance_tags': append_strings(json_alert['performance_tags']) } return alert_characteristics def parse_args(): - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -