in experiments/conductor/cmd/runner/runner.go [211:446]
func RunRunner(ctx context.Context, opts *RunnerOptions) error {
log.Printf("Running conductor runner with branch config: %s", opts.branchConfFile)
if opts.loggingDir == "" {
return fmt.Errorf("logging-dir is required")
}
// If loggingDir is a relative path, make it absolute by prepending the current working directory
if !filepath.IsAbs(opts.loggingDir) {
cwd, err := os.Getwd()
if err != nil {
return fmt.Errorf("getting current working directory: %w", err)
}
opts.loggingDir = filepath.Join(cwd, opts.loggingDir)
}
if err := opts.validateFlags(); err != nil {
return err
}
log.Printf("Starting Runner")
var branches Branches
{
b, err := os.ReadFile(opts.branchConfFile) // For read access.
if err != nil {
return fmt.Errorf("reading branch config file %q: %w", opts.branchConfFile, err)
}
if err := yaml.Unmarshal(b, &branches); err != nil {
return fmt.Errorf("parsing branch config file %q: %w", opts.branchConfFile, err)
}
}
// Filter based on forResources if specified
if opts.forResources != "" {
targetBranches := make(map[string]bool)
for _, name := range strings.Split(opts.forResources, ",") {
targetBranches[strings.TrimSpace(name)] = true
}
var resourceFilteredBranches []Branch
for _, branch := range branches.Branches {
if targetBranches[branch.Name] {
resourceFilteredBranches = append(resourceFilteredBranches, branch)
}
}
branches.Branches = resourceFilteredBranches
}
if opts.forResourcesRegex != "" {
resourceFilteredBranches := make([]Branch, 0)
for _, branch := range branches.Branches {
match, err := regexp.MatchString(opts.forResourcesRegex, branch.Name)
if err != nil {
return fmt.Errorf("error matching regex %q: %w", opts.forResourcesRegex, err)
}
if match {
resourceFilteredBranches = append(resourceFilteredBranches, branch)
}
}
branches.Branches = resourceFilteredBranches
}
// Only filter skipped branches if not running metadata commands
if opts.command >= 0 {
// Filter out skipped branches
var filteredBranches []Branch
for _, branch := range branches.Branches {
if !branch.Skip {
filteredBranches = append(filteredBranches, branch)
}
}
branches.Branches = filteredBranches
}
switch opts.command {
case -6:
if err := filterMetadata(opts, branches); err != nil {
log.Fatalf("unnable to filter metadata: %v", err)
}
case cmdDeleteGitBranch: // -5
for idx, branch := range branches.Branches {
log.Printf("Delete GitHub Branch: %d name: %s, branch: %s\r\n", idx, branch.Name, branch.Local)
deleteGithubBranch(opts, branch)
}
case -4:
fixMetadata(opts, branches, "Skip(with reason)", setSkipOnBranchModifier)
case -3:
fixMetadata(opts, branches, "EnableAPIs", addEnableAPIsModifier)
case -2:
splitMetadata(opts, branches)
case -1:
fixMetadata(opts, branches, "ProtoPath", inferProtoPathModifier)
case cmdHelp: // 0
printHelp()
case cmdCheckRepo: // 1
checkRepoDir(ctx, opts, branches)
case cmdCreateGitBranch: // 2
for idx, branch := range branches.Branches {
/*
if branch.Controller != "Unknown" {
// Skipping TF, DCL and Direct controller resources.
continue
}
*/
log.Printf("Create GitHub Branch: %d name: %s, branch: %s\r\n", idx, branch.Name, branch.Local)
createGithubBranch(opts, branch)
}
case cmdMergeMetadata: // 3
if err := mergeMetadata(opts, branches); err != nil {
log.Fatalf("unnable to merge metadata: %v", err)
}
case cmdEnableGCPAPIs: // 4
for idx, branch := range branches.Branches {
log.Printf("Enable GCP APIs: %d name: %s, branch: %s\r\n", idx, branch.Name, branch.Local)
enableAPIs(opts, branch)
}
case cmdReadFiles: // 5
readFuncs := map[string]func(*RunnerOptions, Branch){
typeScriptYaml: readScriptYaml,
typeHttpLog: readHttpLog,
typeMockGo: readMockGo,
}
if readFunc, ok := readFuncs[strings.ToLower(opts.readFileType)]; ok {
for idx, branch := range branches.Branches {
log.Printf("Read %s: %d name: %s, branch: %s\r\n", opts.readFileType, idx, branch.Name, branch.Local)
readFunc(opts, branch)
}
}
case cmdWriteFiles: // 6
writeFuncs := map[string]func(*RunnerOptions, Branch){
typeScriptYaml: writeScriptYaml,
}
if writeFunc, ok := writeFuncs[strings.ToLower(opts.readFileType)]; ok {
for idx, branch := range branches.Branches {
log.Printf("Write %s: %d name: %s, branch: %s\r\n", opts.readFileType, idx, branch.Name, branch.Local)
writeFunc(opts, branch)
}
}
case cmdDiff: // 7
for idx, branch := range branches.Branches {
log.Printf("Showing diff for last %d commits: %d name: %s, branch: %s\r\n", opts.numCommits, idx, branch.Name, branch.Local)
diffLastNCommits(opts, branch)
}
case cmdRevert: // 8
if opts.numCommits <= 0 {
log.Printf("Skipping revert, num-commits must be positive (got %d)", opts.numCommits)
return nil
}
for idx, branch := range branches.Branches {
log.Printf("Reverting last %d commits in branch %s: %d name: %s, branch: %s\r\n", opts.numCommits, branch.Name, idx, branch.Name, branch.Local)
revertLastNCommits(opts, branch)
}
case cmdPushBranch: // 9
processors := []BranchProcessor{
{Fn: runAPIChecks, CommitMsgTemplate: "{{kind}}: Normalize api checks", AttemptsOnNoChange: 1},
{Fn: makeReadyPR, CommitMsgTemplate: "make ready-pr", AttemptsOnChanges: 5, AttemptsOnNoChange: 1},
{Fn: pushBranch, CommitMsgTemplate: "push branch", AttemptsOnNoChange: 1},
}
processBranches(ctx, opts, branches.Branches, "Prep and Push Branch", processors)
case cmdCreateScriptYaml: // 10
processBranches(ctx, opts, branches.Branches, "Script YAML", []BranchProcessor{{Fn: createScriptYaml, CommitMsgTemplate: "mockgcp: create test script for {{command}}"}})
case cmdCaptureHttpLog: // 11
processBranches(ctx, opts, branches.Branches, "HTTP Log", []BranchProcessor{{Fn: captureHttpLog, CommitMsgTemplate: "mockgcp: golden output for TestScripts/mock{{group}}/testdata/{{resource}}/crud"}})
case cmdGenerateMockGo: // 12
processBranches(ctx, opts, branches.Branches, "Mock Go Files", []BranchProcessor{{Fn: generateMockGo, CommitMsgTemplate: "{{kind}}: Add generated mock files"}})
case cmdAddServiceRoundTrip: // 13
processBranches(ctx, opts, branches.Branches, "Service RoundTrip", []BranchProcessor{{Fn: addServiceToRoundTrip, CommitMsgTemplate: "mockgcp: Add mock{{group}} service to mock_http_roundtrip.go"}})
case cmdAddProtoMakefile: // 14
processBranches(ctx, opts, branches.Branches, "Proto Makefile", []BranchProcessor{{Fn: addProtoToMakefile, CommitMsgTemplate: "{{group}}: Add proto generation to makefile", AttemptsOnNoChange: 1}})
case cmdBuildProto: // 15
processBranches(ctx, opts, branches.Branches, "Build Proto", []BranchProcessor{{Fn: buildProtoFiles, CommitMsgTemplate: "chore: Build and add generated proto files"}})
case cmdCaptureMockOutput: // 16
processBranches(ctx, opts, branches.Branches, "Mock Tests", []BranchProcessor{{Fn: runMockgcpTests, CommitMsgTemplate: "{{kind}}: Capture mock golden output"}})
case cmdRunAndFixMockTests: // 17
processBranches(ctx, opts, branches.Branches, "Mock Tests", []BranchProcessor{{Fn: fixMockgcpFailures, CommitMsgTemplate: "Verify and Fix mock tests", VerifyFn: runMockgcpTests, VerifyAttempts: 10, AttemptsOnNoChange: 2}})
case cmdGenerateTypes: // 20
processBranches(ctx, opts, branches.Branches, "Types", []BranchProcessor{{Fn: generateTypes, CommitMsgTemplate: "{{kind}}: Add generated types"}})
case cmdAdjustTypes: // 21
processors := []BranchProcessor{
{Fn: setTypeSpecStatus, CommitMsgTemplate: "{{kind}}: Add spec and status to generated type", AttemptsOnNoChange: 6},
{Fn: setTypeParent, CommitMsgTemplate: "{{kind}}: Add parent to generated type", AttemptsOnNoChange: 6},
{Fn: adjustIdentityParent, CommitMsgTemplate: "{{kind}}: Adjust identity parent", AttemptsOnNoChange: 6},
{Fn: regenerateTypes, CommitMsgTemplate: "{{kind}}: Regenerate types"},
{Fn: removeNameField, CommitMsgTemplate: "{{kind}}: Remove Name Field"},
{Fn: moveEtagField, CommitMsgTemplate: "{{kind}}: Move Etag Field", AttemptsOnNoChange: 1},
// add a kubebuilder required field label to the fields that are marked required in proto
{Fn: addRequiredFieldTags, CommitMsgTemplate: "{{kind}}: Add Required Field Tags"},
// TODO? manual: Add something to handle references to other resources: https://github.com/GoogleCloudPlatform/k8s-config-connector/pull/4010/commits/1651a0a7af5bca37b5c2e134dd3f600ebac6a172
// * https://github.com/GoogleCloudPlatform/k8s-config-connector/pull/4017/commits/cc726106aff55d41e6bc94272acc3612f2636397
}
processBranches(ctx, opts, branches.Branches, "Adjusting types", processors)
case cmdGenerateCRD: // 22
processBranches(ctx, opts, branches.Branches, "CRD", []BranchProcessor{{Fn: generateCRD, CommitMsgTemplate: "{{kind}}: Add generated CRD"}})
case cmdGenerateMapper: // 23
processBranches(ctx, opts, branches.Branches, "Mapper", []BranchProcessor{{Fn: generateMapper, CommitMsgTemplate: "{{kind}}: Add generated mapper"}})
case cmdGenerateFuzzer: // 24
processBranches(ctx, opts, branches.Branches, "Fuzzer", []BranchProcessor{{Fn: generateFuzzer, CommitMsgTemplate: "{{kind}}: Add generated fuzzer"}})
case cmdRunAndFixFuzzTests: // 25
processBranches(ctx, opts, branches.Branches, "Fuzzer Tests", []BranchProcessor{{Fn: fixFuzzerFailures, CommitMsgTemplate: "{{kind}}: Verify and Fix fuzzer tests", VerifyFn: runFuzzerTests, VerifyAttempts: 5, AttemptsOnNoChange: 2}})
case cmdRunAndFixAPIChecks: // 26
processBranches(ctx, opts, branches.Branches, "API Checks", []BranchProcessor{{Fn: fixAPICheckFailures, CommitMsgTemplate: "{{kind}}: Verify and Fix API checks", VerifyFn: runAPIChecks, VerifyAttempts: 5}})
case cmdControllerClient: // 40
processBranches(ctx, opts, branches.Branches, "Controller Client", []BranchProcessor{{Fn: generateControllerClient, CommitMsgTemplate: "{{kind}}: Add controller client"}})
case cmdGenerateController: // 41
processBranches(ctx, opts, branches.Branches, "Controller", []BranchProcessor{{Fn: generateController, CommitMsgTemplate: "{{kind}}: Add controller"}})
case cmdBuildAndFixController: // 42
processBranches(ctx, opts, branches.Branches, "Build and Fix Controller", []BranchProcessor{{Fn: fixControllerBuild, CommitMsgTemplate: "{{kind}}: Build and fix controller", VerifyFn: buildController, VerifyAttempts: 10}})
case cmdCreateIdentity: // 43
processBranches(ctx, opts, branches.Branches, "Identity and Reference", []BranchProcessor{
{Fn: generateControllerIdentity, CommitMsgTemplate: "{{kind}}: Add controller identity"},
{Fn: generateControllerReference, CommitMsgTemplate: "{{kind}}: Add controller reference"},
})
case cmdControllerCreateTest: // 44
processBranches(ctx, opts, branches.Branches, "Controller Test", []BranchProcessor{
{Fn: createControllerTest, CommitMsgTemplate: "{{kind}}: Create minimal test"},
{Fn: updateTestHarness, CommitMsgTemplate: "{{kind}}: Support for testing with mockgcp"},
})
case cmdCaptureGoldenRealGCPOutput: // 45
processBranches(ctx, opts, branches.Branches, "Record Golden Real GCP Tests", []BranchProcessor{{Fn: runGoldenRealGCPTests, CommitMsgTemplate: "{{kind}}: Record golden logs for real GCP tests", AttemptsOnNoChange: 1}})
case cmdRunAndFixGoldenRealGCPOutput: // 46
processBranches(ctx, opts, branches.Branches, "Fix Real GCP Tests", []BranchProcessor{{Fn: fixGoldenTests, CommitMsgTemplate: "{{kind}}: Verify and Fix real GCP tests", VerifyFn: runGoldenRealGCPTests, VerifyAttempts: 5, AttemptsOnNoChange: 2}})
case cmdCaptureGoldenMockOutput: // 47
processBranches(ctx, opts, branches.Branches, "Record Golden Mock Tests", []BranchProcessor{{Fn: runGoldenMockTests, CommitMsgTemplate: "{{kind}}: Record golden logs for mock GCP tests", AttemptsOnNoChange: 1}})
case cmdRunAndFixGoldenMockOutput: // 48
processBranches(ctx, opts, branches.Branches, "Fix Mock GCP Tests", []BranchProcessor{{Fn: fixMockGcpForGoldenTests, CommitMsgTemplate: "{{kind}}: Verify and Fix mock GCP tests", VerifyFn: runGoldenMockTests, VerifyAttempts: 5, AttemptsOnNoChange: 2}})
default:
log.Fatalf("unrecognized command: %d", opts.command)
}
return nil
}