import logging import sys from treeherder.model.models import ( BugzillaComponent, BugzillaSecurityGroup, FilesBugzillaMap, Repository, ) from treeherder.utils.github import fetch_json logger = logging.getLogger(__name__) class FilesBugzillaMapProcess: bugzilla_components = {} max_path_length = FilesBugzillaMap._meta.get_field("path").max_length max_product_length = BugzillaComponent._meta.get_field("product").max_length max_component_length = BugzillaComponent._meta.get_field("component").max_length run_id = None def get_or_add_bugzilla_component(self, files_bugzilla_data, path): product_component_data = files_bugzilla_data[path] product_component_str = product_component_data[0] + " :: " + product_component_data[1] if product_component_str in self.bugzilla_components: return self.bugzilla_components[product_component_str] try: product = product_component_data[0] component = product_component_data[1] if len(product) > self.max_product_length: logger.error( "error inserting Bugzilla product and component \"'%s' :: '%s'\" into db (file skipped: '%s'): product is too long (has %d characters, max is %d)", product, component, path, len(product), self.max_product_length, ) return if len(component) > self.max_component_length: logger.error( "error inserting Bugzilla product and component \"'%s' :: '%s'\" into db (file skipped: '%s'): component is too long (has %d characters, max is %d)", product, component, path, len(component), self.max_component_length, ) return if len(path) > self.max_path_length: logger.error( "error inserting Bugzilla product and component \"'%s' :: '%s'\" into db (file skipped: '%s'): path is too long (has %d characters, max is %d)", product, component, path, len(path), self.max_path_length, ) bugzilla_component_data, _ = BugzillaComponent.objects.get_or_create( product=product, component=component, ) self.bugzilla_components[product_component_str] = bugzilla_component_data except Exception as e: logger.error( "error inserting Bugzilla product and component \"'%s' :: '%s'\" into db (file skipped: '%s'): %s", product, component, path, e, ) return return bugzilla_component_data def get_projects_to_import(self): return list( Repository.objects.filter(codebase="gecko") .filter(active_status="active") .filter(life_cycle_order__isnull=False) .values_list("name", flat=True) .order_by("life_cycle_order") ) def fetch_data(self, project): url = f"https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/gecko.v2.{project}.latest.source.source-bugzilla-info/artifacts/public/components.json" files_bugzilla_data = None exception = None try: files_bugzilla_data = fetch_json(url) except Exception as e: exception = e return { "url": url, "files_bugzilla_data": files_bugzilla_data, "exception": exception, } def run(self): projects = self.get_projects_to_import() paths_ingested_all = set() paths_bugzilla_ingested_all = set() for project in projects: data_returned = self.fetch_data(project) if data_returned["exception"] is not None: logger.error( "error fetching file with map of source paths to Bugzilla products and components: url: %s ; %s", data_returned["url"], data_returned["exception"], ) continue files_bugzilla_data = data_returned["files_bugzilla_data"] paths_ingested_this_project = set(path for path in files_bugzilla_data).difference( paths_ingested_all ) paths_bugzilla_ingested_project = set() for path in paths_ingested_this_project: paths_bugzilla_ingested_project.add( ( path, files_bugzilla_data[path][0], files_bugzilla_data[path][1], ) ) paths_ingested_all |= paths_ingested_this_project paths_bugzilla_ingested_all |= paths_bugzilla_ingested_project paths_old = set(FilesBugzillaMap.objects.values_list("path", flat=True)) paths_removed = paths_old - paths_ingested_all FilesBugzillaMap.objects.filter(path__in=paths_removed).delete() paths_bugzilla_old = set( FilesBugzillaMap.objects.select_related("bugzilla_component").values_list( "path", "bugzilla_component__product", "bugzilla_component__component" ) ) paths_bugzilla_unchanged = paths_bugzilla_old.intersection(paths_bugzilla_ingested_all) paths_bugzilla_changed_or_added = paths_bugzilla_ingested_all.difference( paths_bugzilla_unchanged ) paths_changed_or_added = set( path_bugzilla[0] for path_bugzilla in paths_bugzilla_changed_or_added ) paths_added = paths_ingested_all.difference(paths_old) paths_changed = paths_changed_or_added.difference(paths_added) path_bugzilla_data = {} for path_bugzilla in paths_bugzilla_ingested_all: path = path_bugzilla[0] product = path_bugzilla[1] component = path_bugzilla[2] path_bugzilla_data[path] = [product, component] paths_bugzilla_update_needed = [] for path in paths_changed: bugzilla_component_data = self.get_or_add_bugzilla_component(path_bugzilla_data, path) if not bugzilla_component_data: continue path_bugzilla_update_needed = FilesBugzillaMap.objects.select_related( "bugzilla_component" ).filter(path=path)[0] path_bugzilla_update_needed.bugzilla_component_id = bugzilla_component_data.id paths_bugzilla_update_needed.append(path_bugzilla_update_needed) FilesBugzillaMap.objects.bulk_update( paths_bugzilla_update_needed, ["bugzilla_component_id"], batch_size=1000 ) paths_bugzilla_addition_needed = [] for path in paths_added: bugzilla_component_data = self.get_or_add_bugzilla_component(path_bugzilla_data, path) if not bugzilla_component_data: continue file_name = (path.rsplit("/", 1))[-1] paths_bugzilla_addition_needed.append( FilesBugzillaMap( path=path, file_name=file_name, bugzilla_component=bugzilla_component_data, ) ) FilesBugzillaMap.objects.bulk_create(paths_bugzilla_addition_needed, batch_size=1000) bugzilla_components_used = set( FilesBugzillaMap.objects.values_list("bugzilla_component_id", flat=True).distinct() ) bugzilla_components_all = set( BugzillaComponent.objects.all().values_list("id", flat=True).distinct() ) bugzilla_components_unused = bugzilla_components_all.difference(bugzilla_components_used) (BugzillaComponent.objects.filter(id__in=bugzilla_components_unused).delete()) class ProductSecurityGroupProcess: max_product_length = BugzillaSecurityGroup._meta.get_field("product").max_length max_security_group_length = BugzillaSecurityGroup._meta.get_field("security_group").max_length def fetch_data(self): url = "https://bugzilla.mozilla.org/latest/configuration" product_security_group_data = None exception = None try: product_security_group_data = fetch_json(url) except Exception as e: exception = e return { "url": url, "product_security_group_data": product_security_group_data, "exception": exception, } def run(self): data_returned = self.fetch_data() if data_returned["exception"] is not None: logger.error( "error fetching file with map of source paths to Bugzilla products and components: url: %s ; %s", data_returned["url"], data_returned["exception"], ) sys.exit() fields_data = data_returned["product_security_group_data"]["field"]["product"]["values"] groups_data = data_returned["product_security_group_data"]["group"] products = set() for field_data in fields_data: product_name = str(field_data["name"]) security_group_id = str(field_data["security_group_id"]) if security_group_id in groups_data: security_group_name = str(groups_data[security_group_id]["name"]) products.add(product_name) try: if len(product_name) > self.max_product_length: logger.error( "error inserting Bugzilla product and security group \"'%s' :: '%s'\" into db: product is too long (has %d characters, max is %d)", product_name, security_group_name, len(product_name), self.max_product_length, ) continue if len(security_group_name) > self.max_security_group_length: logger.error( "error inserting Bugzilla product and security group \"'%s' :: '%s'\" into db: security group is too long (has %d characters, max is %d)", product_name, security_group_name, len(security_group_name), self.max_security_group_length, ) continue BugzillaSecurityGroup.objects.get_or_create( product=product_name, security_group=security_group_name, ) except Exception as e: logger.error( "error inserting Bugzilla product and security group \"'%s' :: '%s'\" into db: %s", product_name, security_group_name, e, ) continue BugzillaSecurityGroup.objects.exclude(product__in=products).delete()