eng/tools/generator/cmd/v2/automation/automationCmd.go (287 lines of code) (raw):
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License. See License.txt in the project root for license information.
package automation
import (
"archive/zip"
"errors"
"fmt"
"io"
"io/fs"
"log"
"os"
"path/filepath"
"strings"
"github.com/Azure/azure-sdk-for-go/eng/tools/generator/cmd/v2/automation/pipeline"
"github.com/Azure/azure-sdk-for-go/eng/tools/generator/cmd/v2/common"
"github.com/Azure/azure-sdk-for-go/eng/tools/generator/repo"
"github.com/Azure/azure-sdk-for-go/eng/tools/generator/typespec"
"github.com/Azure/azure-sdk-for-go/eng/tools/internal/utils"
"github.com/spf13/cobra"
)
// Command returns the automation v2 command. Note that this command is designed to run in the root directory of
// azure-sdk-for-go. It does not work if you are running this tool in somewhere else
func Command() *cobra.Command {
cmd := &cobra.Command{
Use: "automation-v2 <generate input filepath> <generate output filepath> [goVersion]",
Args: cobra.RangeArgs(2, 3),
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
log.SetFlags(0) // remove the time stamp prefix
log.SetOutput(os.Stdout) // set the output to stdout
cmd.SetErrPrefix("[ERROR]")
return nil
},
RunE: func(cmd *cobra.Command, args []string) error {
goVersion := "1.18"
if len(args) == 3 {
goVersion = args[2]
}
if err := execute(args[0], args[1], goVersion); err != nil {
return errors.New(logError(err))
}
return nil
},
SilenceUsage: true, // this command is used for a pipeline, the usage should never show
}
return cmd
}
func execute(inputPath, outputPath, goVersion string) error {
log.Printf("Reading generate input file from '%s'...", inputPath)
input, err := pipeline.ReadInput(inputPath)
if err != nil {
return fmt.Errorf("cannot read generate input: %+v", err)
}
cwd, err := os.Getwd()
if err != nil {
return err
}
log.Printf("Using current directory as SDK root: %s", cwd)
ctx := automationContext{
sdkRoot: utils.NormalizePath(cwd),
specRoot: input.SpecFolder,
commitHash: input.HeadSha,
goVersion: goVersion,
}
output, err := ctx.generate(input)
if output != nil && len(output.Packages) != 0 {
log.Printf("Writing output to file '%s'...", outputPath)
if err := pipeline.WriteOutput(outputPath, output); err != nil {
return fmt.Errorf("cannot write generate output: %+v", err)
}
}
if err != nil {
return err
}
return nil
}
type automationContext struct {
sdkRoot string
specRoot string
commitHash string
goVersion string
}
// TODO -- support dry run
func (ctx *automationContext) generate(input *pipeline.GenerateInput) (*pipeline.GenerateOutput, error) {
if input.DryRun {
return nil, fmt.Errorf("dry run not supported yet")
}
// iterate over all the readme
results := make([]pipeline.PackageResult, 0)
errorBuilder := generateErrorBuilder{}
// create sdk repo ref
sdkRepo, err := repo.OpenSDKRepository(ctx.sdkRoot)
if err != nil {
return nil, fmt.Errorf("failed to get sdk repo: %+v", err)
}
// typespec
// Generated by tsp only when tspconfig.yaml exists and has typespec-go option
for _, tspProjectFolder := range input.RelatedTypeSpecProjectFolder {
tspconfigPath := filepath.Join(input.SpecFolder, tspProjectFolder, "tspconfig.yaml")
tsc, err := typespec.ParseTypeSpecConfig(tspconfigPath)
if err != nil {
errorBuilder.add(fmt.Errorf("failed to parse %s: %+v\nInvalid tspconfig.yaml provided and refer to the sample file: https://aka.ms/azsdk/tspconfig-sample-mpg to fix the content", tspconfigPath, err))
continue
}
if ok := tsc.ExistEmitOption(string(typespec.TypeSpec_GO)); !ok {
log.Println(fmt.Sprintf("`@azure-tools/typespec-go` option not found in %s, it is required, please refer to `https://aka.ms/azsdk/tspconfig-sample-mpg` to configure it", tspconfigPath))
continue
}
log.Printf("Start to process typespec project: %s", tspProjectFolder)
generateCtx := common.GenerateContext{
SDKPath: sdkRepo.Root(),
SDKRepo: &sdkRepo,
SpecPath: ctx.specRoot,
SpecCommitHash: ctx.commitHash,
SpecRepoURL: input.RepoHTTPSURL,
TypeSpecConfig: tsc,
}
namespaceResult, err := generateCtx.GenerateForTypeSpec(&common.GenerateParam{
SkipGenerateExample: true,
GoVersion: ctx.goVersion,
TspClientOptions: []string{"--debug"},
})
if err != nil {
errorBuilder.add(err)
continue
}
content := namespaceResult.ChangelogMD
breaking := namespaceResult.Changelog.HasBreakingChanges()
if namespaceResult.PullRequestLabels == string(common.FirstGABreakingChangeLabel) || namespaceResult.PullRequestLabels == string(common.BetaBreakingChangeLabel) {
// If the PR is first beta or first GA, it is not necessary to report SDK breaking change in spec PR
breaking = false
}
breakingChangeItems := namespaceResult.Changelog.GetBreakingChangeItems()
packageRelativePath := namespaceResult.PackageRelativePath
srcFolder := filepath.Join(sdkRepo.Root(), packageRelativePath)
apiViewArtifact := filepath.Join(sdkRepo.Root(), packageRelativePath+".gosource")
err = zipDirectory(srcFolder, apiViewArtifact)
if err != nil {
fmt.Println(err)
}
results = append(results, pipeline.PackageResult{
Version: namespaceResult.Version,
PackageName: packageRelativePath,
Path: []string{packageRelativePath},
PackageFolder: packageRelativePath,
TypespecProject: []string{tspProjectFolder},
Changelog: &pipeline.Changelog{
Content: &content,
HasBreakingChange: &breaking,
BreakingChangeItems: &breakingChangeItems,
},
APIViewArtifact: packageRelativePath + ".gosource",
Language: "Go",
})
log.Printf("Finish processing typespec file: %s", tspconfigPath)
}
// autorest
if input.RelatedReadmeMdFile != "" {
input.RelatedReadmeMdFiles = append(input.RelatedReadmeMdFiles, input.RelatedReadmeMdFile)
}
for _, readme := range input.RelatedReadmeMdFiles {
log.Printf("Start to process autorest project: %s", readme)
sepStrs := strings.Split(readme, "/")
for i, sepStr := range sepStrs {
if sepStr == "resource-manager" {
readme = strings.Join(sepStrs[i-1:], "/")
if i > 1 {
ctx.specRoot = input.SpecFolder + "/" + strings.Join(sepStrs[:i-1], "/")
}
break
}
}
generateCtx := common.GenerateContext{
SDKPath: sdkRepo.Root(),
SDKRepo: &sdkRepo,
SpecPath: ctx.specRoot,
}
namespaceResults, errors := generateCtx.GenerateForAutomation(readme, input.RepoHTTPSURL, ctx.goVersion)
if len(errors) != 0 {
errorBuilder.add(errors...)
continue
}
for _, namespaceResult := range namespaceResults {
content := namespaceResult.ChangelogMD
breaking := namespaceResult.Changelog.HasBreakingChanges()
if namespaceResult.PullRequestLabels == string(common.FirstGABreakingChangeLabel) || namespaceResult.PullRequestLabels == string(common.BetaBreakingChangeLabel) {
// If the pr is beta or first GA, it's no necessary to report sdk breaking change in spec pr
breaking = false
}
breakingChangeItems := namespaceResult.Changelog.GetBreakingChangeItems()
srcFolder := filepath.Join(sdkRepo.Root(), "sdk", "resourcemanager", namespaceResult.RPName, namespaceResult.PackageName)
apiViewArtifact := filepath.Join(sdkRepo.Root(), "sdk", "resourcemanager", namespaceResult.RPName, namespaceResult.PackageName+".gosource")
err := zipDirectory(srcFolder, apiViewArtifact)
if err != nil {
fmt.Println(err)
}
results = append(results, pipeline.PackageResult{
Version: namespaceResult.Version,
PackageName: fmt.Sprintf("sdk/resourcemanager/%s/%s", namespaceResult.RPName, namespaceResult.PackageName),
Path: []string{fmt.Sprintf("sdk/resourcemanager/%s/%s", namespaceResult.RPName, namespaceResult.PackageName)},
PackageFolder: fmt.Sprintf("sdk/resourcemanager/%s/%s", namespaceResult.RPName, namespaceResult.PackageName),
ReadmeMd: []string{readme},
Changelog: &pipeline.Changelog{
Content: &content,
HasBreakingChange: &breaking,
BreakingChangeItems: &breakingChangeItems,
},
APIViewArtifact: fmt.Sprintf("sdk/resourcemanager/%s/%s", namespaceResult.RPName, namespaceResult.PackageName+".gosource"),
Language: "Go",
})
}
log.Printf("Finish processing readme file: %s", readme)
}
return &pipeline.GenerateOutput{
Packages: results,
}, errorBuilder.build()
}
type generateErrorBuilder struct {
errors []error
}
func (b *generateErrorBuilder) add(err ...error) {
b.errors = append(b.errors, err...)
}
func (b *generateErrorBuilder) build() error {
if len(b.errors) == 0 {
return nil
}
var messages []string
for _, err := range b.errors {
messages = append(messages, err.Error())
}
return fmt.Errorf("total %d error(s): \n%s\n%s", len(b.errors), strings.Join(messages, "\n"), `Refer to the detail errors for potential fixes.
If the issue persists, contact the Go language support channel at https://aka.ms/azsdk/go-lang-teams-channel and include this spec pull request.`)
}
func logError(err error) string {
buidler := strings.Builder{}
for i, line := range strings.Split(err.Error(), "\n") {
if l := strings.TrimSpace(line); l != "" {
if i == 0 {
buidler.WriteString(fmt.Sprintf("%s\n", l))
continue
}
buidler.WriteString(fmt.Sprintf("[ERROR] %s\n", l))
}
}
return buidler.String()
}
func zipDirectory(srcFolder, dstZip string) error {
outFile, err := os.Create(dstZip)
if err != nil {
return err
}
w := zip.NewWriter(outFile)
srcFolder = strings.TrimSuffix(srcFolder, string(os.PathSeparator))
err = filepath.Walk(srcFolder, func(path string, info fs.FileInfo, err error) error {
if err != nil {
return err
}
header, err := zip.FileInfoHeader(info)
if err != nil {
return err
}
header.Method = zip.Deflate
header.Name, err = filepath.Rel(filepath.Dir(srcFolder), path)
if err != nil {
return err
}
if info.IsDir() {
header.Name += string(os.PathSeparator)
}
hw, err := w.CreateHeader(header)
if err != nil {
return err
}
if info.IsDir() {
return nil
}
f, err := os.Open(path)
if err != nil {
return err
}
_, err = io.Copy(hw, f)
if err != nil {
return err
}
return nil
})
if err != nil {
return err
}
err = w.Close()
if err != nil {
return err
}
err = outFile.Close()
if err != nil {
return err
}
return nil
}