backend/plugins/gitlab/tasks/mr

/* Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package tasks import ( "regexp" "strings" "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/common" "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/gitlab/models" ) func init() { RegisterSubtaskMeta(&ExtractApiMergeRequestsMeta) } type MergeRequestRes struct { GitlabId int `json:"id"` Iid int ProjectId int `json:"project_id"` SourceProjectId int `json:"source_project_id"` TargetProjectId int `json:"target_project_id"` State string Title string Description string WebUrl string `json:"web_url"` UserNotesCount int `json:"user_notes_count"` WorkInProgress bool `json:"work_in_progress"` SourceBranch string `json:"source_branch"` TargetBranch string `json:"target_branch"` GitlabCreatedAt common.Iso8601Time `json:"created_at"` GitlabUpdatedAt *common.Iso8601Time `json:"updated_at"` MergedAt *common.Iso8601Time `json:"merged_at"` ClosedAt *common.Iso8601Time `json:"closed_at"` MergeCommitSha string `json:"merge_commit_sha"` SquashCommitSha string `json:"squash_commit_sha"` DiffHeadSha string `json:"sha"` MergedBy struct { Username string `json:"username"` } `json:"merged_by"` Author struct { Id int `json:"id"` Username string `json:"username"` } Reviewers []Reviewer Assignees []Assignee FirstCommentTime common.Iso8601Time Labels []string `json:"labels"` } type Reviewer struct { ReviewerIdId int `json:"id"` MergeRequestId int Name string Username string State string AvatarUrl string `json:"avatar_url"` WebUrl string `json:"web_url"` } type Assignee struct { AssigneeId int `json:"id"` MergeRequestId int Name string Username string State string AvatarUrl string `json:"avatar_url"` WebUrl string `json:"web_url"` } var ExtractApiMergeRequestsMeta = plugin.SubTaskMeta{ Name: "Extract Merge Requests", EntryPoint: ExtractApiMergeRequests, EnabledByDefault: true, Description: "Extract raw merge requests data into tool layer table GitlabMergeRequest and GitlabReviewer", DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_REVIEW}, Dependencies: []*plugin.SubTaskMeta{&CollectApiMergeRequestsMeta}, } func ExtractApiMergeRequests(subtaskCtx plugin.SubTaskContext) errors.Error { subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_MERGE_REQUEST_TABLE) db := subtaskCtx.GetDal() config := data.Options.ScopeConfig var labelTypeRegex *regexp.Regexp var labelComponentRegex *regexp.Regexp var prType = config.PrType var err1 error if len(prType) > 0 { labelTypeRegex, err1 = regexp.Compile(prType) if err1 != nil { return errors.Default.Wrap(err1, "regexp Compile prType failed") } } var prComponent = config.PrComponent if len(prComponent) > 0 { labelComponentRegex, err1 = regexp.Compile(prComponent) if err1 != nil { return errors.Default.Wrap(err1, "regexp Compile prComponent failed") } } subtaskCommonArgs.SubtaskConfig = map[string]any{ "prType": prType, "prComponent": prComponent, } extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs[MergeRequestRes]{ SubtaskCommonArgs: subtaskCommonArgs, BeforeExtract: beforeExtractMr(db, data), Extract: func(mr *MergeRequestRes, row *api.RawData) ([]interface{}, errors.Error) { gitlabMergeRequest, err := convertMergeRequest(mr) if err != nil { return nil, err } // if we can not find merged_at and closed_at info in the detail // we need get detail for gitlab v11 s := string(row.Data) if !strings.Contains(s, "\"merged_at\":") { if !strings.Contains(s, "\"closed_at\":") { gitlabMergeRequest.IsDetailRequired = true } } results := make([]interface{}, 0, len(mr.Reviewers)+len(mr.Labels)+1) gitlabMergeRequest.ConnectionId = data.Options.ConnectionId results = append(results, gitlabMergeRequest) err = db.Delete( &models.GitlabMrLabel{}, dal.Where("connection_id = ? AND mr_id = ?", data.Options.ConnectionId, gitlabMergeRequest.GitlabId), ) if err != nil { return nil, err } for _, label := range mr.Labels { results = append(results, &models.GitlabMrLabel{ MrId: gitlabMergeRequest.GitlabId, LabelName: label, ConnectionId: data.Options.ConnectionId, }) // if pr.Type has not been set and prType is set in .env, process the below if labelTypeRegex != nil && labelTypeRegex.MatchString(label) { gitlabMergeRequest.Type = label } // if pr.Component has not been set and prComponent is set in .env, process if labelComponentRegex != nil && labelComponentRegex.MatchString(label) { gitlabMergeRequest.Component = label } } err = db.Delete( &models.GitlabReviewer{}, dal.Where("connection_id = ? AND merge_request_id = ?", data.Options.ConnectionId, gitlabMergeRequest.GitlabId), ) if err != nil { return nil, err } for _, reviewer := range mr.Reviewers { gitlabReviewer := &models.GitlabReviewer{ ConnectionId: data.Options.ConnectionId, ReviewerId: reviewer.ReviewerIdId, MergeRequestId: mr.GitlabId, ProjectId: data.Options.ProjectId, Username: reviewer.Username, Name: reviewer.Name, State: reviewer.State, AvatarUrl: reviewer.AvatarUrl, WebUrl: reviewer.WebUrl, } results = append(results, gitlabReviewer) } err = db.Delete( &models.GitlabAssignee{}, dal.Where("connection_id = ? AND merge_request_id = ?", data.Options.ConnectionId, gitlabMergeRequest.GitlabId), ) if err != nil { return nil, err } for _, assignee := range mr.Assignees { gitlabAssignee := &models.GitlabAssignee{ ConnectionId: data.Options.ConnectionId, AssigneeId: assignee.AssigneeId, MergeRequestId: mr.GitlabId, ProjectId: data.Options.ProjectId, Username: assignee.Username, Name: assignee.Name, State: assignee.State, AvatarUrl: assignee.AvatarUrl, WebUrl: assignee.WebUrl, } results = append(results, gitlabAssignee) } return results, nil }, }) if err != nil { return err } return extractor.Execute() } func convertMergeRequest(mr *MergeRequestRes) (*models.GitlabMergeRequest, errors.Error) { gitlabMergeRequest := &models.GitlabMergeRequest{ GitlabId: mr.GitlabId, Iid: mr.Iid, ProjectId: mr.ProjectId, SourceProjectId: mr.SourceProjectId, TargetProjectId: mr.TargetProjectId, State: mr.State, Title: mr.Title, Description: mr.Description, WebUrl: mr.WebUrl, UserNotesCount: mr.UserNotesCount, WorkInProgress: mr.WorkInProgress, IsDetailRequired: false, SourceBranch: mr.SourceBranch, TargetBranch: mr.TargetBranch, MergeCommitSha: mr.MergeCommitSha, SquashCommitSha: mr.SquashCommitSha, DiffHeadSha: mr.DiffHeadSha, MergedAt: common.Iso8601TimeToTime(mr.MergedAt), GitlabCreatedAt: mr.GitlabCreatedAt.ToTime(), GitlabUpdatedAt: common.Iso8601TimeToTime(mr.GitlabUpdatedAt), ClosedAt: common.Iso8601TimeToTime(mr.ClosedAt), MergedByUsername: mr.MergedBy.Username, AuthorUsername: mr.Author.Username, AuthorUserId: mr.Author.Id, } return gitlabMergeRequest, nil } func beforeExtractMr(db dal.Dal, data *GitlabTaskData) func(mr *MergeRequestRes, stateManager *api.SubtaskStateManager) errors.Error { return func(mr *MergeRequestRes, stateManager *api.SubtaskStateManager) errors.Error { if stateManager.IsIncremental() { err := db.Delete( &models.GitlabMrLabel{}, dal.Where("connection_id = ? AND mr_id = ?", data.Options.ConnectionId, mr.GitlabId), ) if err != nil { return err } err = db.Delete( &models.GitlabAssignee{}, dal.Where("connection_id = ? AND merge_request_id = ?", data.Options.ConnectionId, mr.GitlabId), ) if err != nil { return err } } return nil } }

backend/plugins/gitlab/tasks/mr_extractor.go (235 lines of code) (raw):