import os import requests import time from utils import fetch_labels_mapping, fetch_allowed_labels, convert_label class Importer: _GITHUB_ISSUE_PREFIX = "INFRA-" _PLACEHOLDER_PREFIX = "@PSTART" _PLACEHOLDER_SUFFIX = "@PEND" _DEFAULT_TIME_OUT = 120.0 def __init__(self, options, project): self.options = options self.project = project self.github_url = 'https://api.github.com/repos/%s/%s' % ( self.options.account, self.options.repo) self.jira_issue_replace_patterns = { 'https://issues.jenkins.io/browse/%s%s' % (self.project.name, r'-(\d+)'): r'\1', self.project.name + r'-(\d+)': Importer._GITHUB_ISSUE_PREFIX + r'\1', r'Issue (\d+)': Importer._GITHUB_ISSUE_PREFIX + r'\1'} self.headers = { 'Accept': 'application/vnd.github.golden-comet-preview+json', 'Authorization': f'token {options.accesstoken}' } self.labels_mapping = fetch_labels_mapping() self.approved_labels = fetch_allowed_labels() def import_milestones(self): """ Imports the gathered project milestones into GitHub and remembers the created milestone ids """ milestone_url = self.github_url + '/milestones' print('Importing milestones...', milestone_url) print # Check existing first existing = list() def get_milestone_list(url): return requests.get(url, headers=self.headers, timeout=Importer._DEFAULT_TIME_OUT) def get_next_page_url(url): return url.replace('<', '').replace('>', '').replace('; rel="next"', '') milestone_pages = list() ms = get_milestone_list(milestone_url + '?state=all') milestone_pages.append(ms.json()) if 'Link' in ms.headers: links = ms.headers['Link'].split(',') nextPageUrl = get_next_page_url(links[0]) while nextPageUrl is not None: time.sleep(1) nextPageUrl = None for l in links: if 'rel="next"' in l: nextPageUrl = get_next_page_url(l) if nextPageUrl is not None: ms = get_milestone_list(nextPageUrl) links = ms.headers['Link'].split(',') milestone_pages.append(ms.json()) for ms_json in milestone_pages: for m in ms_json: print(self.project.get_milestones().keys()) try: if m['title'] in self.project.get_milestones().keys(): self.project.get_milestones()[m['title']] = m['number'] print(m['title'], 'found') existing.append(m['title']) except TypeError: pass # Export new ones for mkey in self.project.get_milestones().keys(): if mkey in existing: continue data = {'title': mkey} r = requests.post(milestone_url, json=data, headers=self.headers, timeout=Importer._DEFAULT_TIME_OUT) # overwrite histogram data with the actual milestone id now if r.status_code == 201: content = r.json() self.project.get_milestones()[mkey] = content['number'] print(mkey) def import_labels(self, colour_selector): """ Imports the gathered project components and labels as labels into GitHub """ label_url = self.github_url + '/labels' print('Importing labels...', label_url) print() for lkey in self.project.get_all_labels().keys(): prefixed_lkey = lkey.lower() # prefix component if os.getenv('JIRA_MIGRATION_INCLUDE_COMPONENT_IN_LABELS', 'true') == 'true': if lkey in self.project.get_components().keys(): prefixed_lkey = 'jira-component:' + prefixed_lkey prefixed_lkey = convert_label(prefixed_lkey, self.labels_mapping, self.approved_labels) if prefixed_lkey is None: continue data = {'name': prefixed_lkey, 'color': colour_selector.get_colour(lkey)} r = requests.post(label_url, json=data, headers=self.headers, timeout=Importer._DEFAULT_TIME_OUT) if r.status_code == 201: print(lkey + '->' + prefixed_lkey) else: print('Failure importing label ' + prefixed_lkey, r.status_code, r.content, r.headers) def import_issues(self, start_from_count): """ Starts the issue import into GitHub: First the milestone id is captured for the issue. Then JIRA issue relationships are converted into comments. After that, the comments are taken out of the issue and references to JIRA issues in comments are replaced with a placeholder """ print('Importing issues...') count = 0 for issue in self.project.get_issues(): if start_from_count > count: count += 1 continue print("Index = ", count) if 'milestone_name' in issue: issue['milestone'] = self.project.get_milestones()[ issue['milestone_name']] del issue['milestone_name'] self.convert_relationships_to_comments(issue) issue_comments = issue['comments'] del issue['comments'] comments = [] for comment in issue_comments: comments.append( dict((k, self._replace_jira_with_github_id(v)) for k, v in comment.items())) self.import_issue_with_comments(issue, comments) count += 1 def import_issue_with_comments(self, issue, comments): """ Imports a single issue with its comments into GitHub. Importing via GitHub's normal Issue API quickly triggers anti-abuse rate limits. So their unofficial Issue Import API is used instead: https://gist.github.com/jonmagic/5282384165e0f86ef105 This is a two-step process: First the issue with the comments is pushed to GitHub asynchronously. Then GitHub is pulled in a loop until the issue import is completed. Finally the issue github is noted. """ print('Issue ', issue['key']) print('Labels', issue['labels']) jira_key = issue['key'] del issue['key'] response = self.upload_github_issue(issue, comments) status_url = response.json()['url'] gh_issue_url = self.wait_for_issue_creation(status_url).json()['issue_url'] gh_issue_id = int(gh_issue_url.split('/')[-1]) issue['githubid'] = gh_issue_id issue['key'] = jira_key jira_gh = f"{jira_key}:{gh_issue_id}\n" with open('jira-keys-to-github-id.txt', 'a') as f: f.write(jira_gh) def upload_github_issue(self, issue, comments): """ Uploads a single issue to GitHub asynchronously with the Issue Import API. """ issue_url = self.github_url + '/import/issues' issue_data = {'issue': issue, 'comments': comments} response = requests.post(issue_url, json=issue_data, headers=self.headers, timeout=Importer._DEFAULT_TIME_OUT) if response.status_code == 202: return response elif response.status_code == 422: raise RuntimeError( "Initial import validation failed for issue '{}' due to the " "following errors:\n{}".format(issue['title'], response.json()) ) else: raise RuntimeError( "Failed to POST issue: '{}' due to unexpected HTTP status code: {}\nerrors:\n{}" .format(issue['title'], response.status_code, response.json()) ) def wait_for_issue_creation(self, status_url): """ Check the status of a GitHub issue import. If the status is 'pending', it sleeps, then rechecks until the status is either 'imported' or 'failed'. """ while True: # keep checking until status is something other than 'pending' time.sleep(3) response = requests.get(status_url, headers=self.headers, timeout=Importer._DEFAULT_TIME_OUT) if response.status_code == 404: continue elif response.status_code != 200: raise RuntimeError( "Failed to check GitHub issue import status url: {} due to unexpected HTTP status code: {}" .format(status_url, response.status_code) ) status = response.json()['status'] if status != 'pending': break if status == 'imported': print("Imported Issue:", response.json()['issue_url'].replace('api.github.com/repos/', 'github.com/')) elif status == 'failed': raise RuntimeError( "Failed to import GitHub issue due to the following errors:\n{}" .format(response.json()) ) else: raise RuntimeError( "Status check for GitHub issue import returned unexpected status: '{}'" .format(status) ) return response def convert_relationships_to_comments(self, issue): duplicates = issue['duplicates'] is_duplicated_by = issue['is-duplicated-by'] relates_to = issue['is-related-to'] depends_on = issue['depends-on'] blocks = issue['blocks'] try: epic_link = issue['epic-link'] except (AttributeError, KeyError): epic_link = None for duplicate_item in duplicates: issue['comments'].append( {"body": f'[Duplicates: ' + self._replace_jira_with_github_id(duplicate_item) + ']'}) for is_duplicated_by_item in is_duplicated_by: issue['comments'].append( {"body": f'[Originally duplicated by: ' + self._replace_jira_with_github_id(is_duplicated_by_item) + ']'}) for relates_to_item in relates_to: issue['comments'].append( {"body": f'[Originally related to: ' + self._replace_jira_with_github_id(relates_to_item) + ']'}) for depends_on_item in depends_on: issue['comments'].append( {"body": f'[Originally depends on: ' + self._replace_jira_with_github_id(depends_on_item) + ']'}) for blocks_item in blocks: issue['comments'].append( {"body": f'[Originally blocks: ' + self._replace_jira_with_github_id(blocks_item) + ']'}) if epic_link: issue['comments'].append( {"body": f'[Epic: ' + self._replace_jira_with_github_id(epic_link) + ']'}) del issue['duplicates'] del issue['is-duplicated-by'] del issue['is-related-to'] del issue['depends-on'] del issue['blocks'] try: del issue['epic-link'] except KeyError: pass def _replace_jira_with_github_id(self, text): result = text # for pattern, replacement in self.jira_issue_replace_patterns.items(): # result = re.sub(pattern, Importer._PLACEHOLDER_PREFIX + # replacement + Importer._PLACEHOLDER_SUFFIX, result) return result # def post_process_comments(self): # """ # Starts post-processing all issue comments. # """ # comment_url = self.github_url + '/issues/comments' # self._post_process_comments(comment_url) # def _post_process_comments(self, url): # """ # Paginates through all issue comments and replaces the issue id placeholders with the correct issue ids. # """ # print("listing comments using " + url) # response = requests.get(url, headers=self.headers, # timeout=Importer._DEFAULT_TIME_OUT) # if response.status_code != 200: # raise RuntimeError( # "Failed to list all comments due to unexpected HTTP status code: {}".format( # response.status_code) # ) # comments = response.json() # for comment in comments: # print("handling comment " + comment['url']) # body = comment['body'] # if Importer._PLACEHOLDER_PREFIX in body: # newbody = self._replace_github_id_placeholder(body) # self._patch_comment(comment['url'], newbody) # try: # next_comments = response.links["next"] # if next_comments: # next_url = next_comments['url'] # self._post_process_comments(next_url) # except KeyError: # print('no more pages for comments: ') # for key, value in response.links.items(): # print(key) # print(value) def _replace_github_id_placeholder(self, text): result = text # pattern = Importer._PLACEHOLDER_PREFIX + Importer._GITHUB_ISSUE_PREFIX + \ # r'(\d+)' + Importer._PLACEHOLDER_SUFFIX # result = re.sub(pattern, Importer._GITHUB_ISSUE_PREFIX + r'\1', result) # pattern = Importer._PLACEHOLDER_PREFIX + \ # r'(\d+)' + Importer._PLACEHOLDER_SUFFIX # result = re.sub(pattern, r'\1', result) return result # def _patch_comment(self, url, body): # """ # Patches a single comment body of a Github issue. # """ # print("patching comment " + url) # # print("new body:" + body) # patch_data = {'body': body} # # print(patch_data) # response = requests.patch(url, json=patch_data, headers=self.headers, # timeout=Importer._DEFAULT_TIME_OUT) # if response.status_code != 200: # raise RuntimeError( # "Failed to patch comment {} due to unexpected HTTP status code: {} ; text: {}".format( # url, response.status_code, response.text) # )