mmv1/provider/terraform_tgc.go (376 lines of code) (raw):

// Copyright 2024 Google Inc. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Code generator for a library converting terraform state to gcp objects. package provider import ( "bytes" "errors" "fmt" "io/ioutil" "log" "os" "path" "path/filepath" "regexp" "strings" "time" "golang.org/x/exp/slices" "github.com/GoogleCloudPlatform/magic-modules/mmv1/api" "github.com/GoogleCloudPlatform/magic-modules/mmv1/api/product" "github.com/GoogleCloudPlatform/magic-modules/mmv1/google" ) type TerraformGoogleConversion struct { IamResources []map[string]string NonDefinedTests []string Tests []string TargetVersionName string Version product.Version Product *api.Product StartTime time.Time } func NewTerraformGoogleConversion(product *api.Product, versionName string, startTime time.Time) TerraformGoogleConversion { t := TerraformGoogleConversion{ Product: product, TargetVersionName: versionName, Version: *product.VersionObjOrClosest(versionName), StartTime: startTime, } t.Product.SetPropertiesBasedOnVersion(&t.Version) for _, r := range t.Product.Objects { r.SetCompiler(ProviderName(t)) r.ImportPath = ImportPathFromVersion(versionName) } return t } func (tgc TerraformGoogleConversion) Generate(outputFolder, productPath, resourceToGenerate string, generateCode, generateDocs bool) { // Temporary shim to generate the missing resources directory. Can be removed // once the folder exists downstream. resourcesFolder := path.Join(outputFolder, "converters/google/resources") if err := os.MkdirAll(resourcesFolder, os.ModePerm); err != nil { log.Println(fmt.Errorf("error creating parent directory %v: %v", resourcesFolder, err)) } tgc.GenerateObjects(outputFolder, resourceToGenerate, generateCode, generateDocs) } func (tgc TerraformGoogleConversion) GenerateObjects(outputFolder, resourceToGenerate string, generateCode, generateDocs bool) { for _, object := range tgc.Product.Objects { object.ExcludeIfNotInVersion(&tgc.Version) if resourceToGenerate != "" && object.Name != resourceToGenerate { log.Printf("Excluding %s per user request", object.Name) continue } tgc.GenerateObject(*object, outputFolder, tgc.TargetVersionName, generateCode, generateDocs) } } func (tgc TerraformGoogleConversion) GenerateObject(object api.Resource, outputFolder, resourceToGenerate string, generateCode, generateDocs bool) { if object.ExcludeTgc { log.Printf("Skipping fine-grained resource %s", object.Name) return } templateData := NewTemplateData(outputFolder, tgc.TargetVersionName) if !object.IsExcluded() { tgc.GenerateResource(object, *templateData, outputFolder, generateCode, generateDocs) if generateCode { // tgc.GenerateResourceTests(object, *templateData, outputFolder) // tgc.GenerateResourceSweeper(object, *templateData, outputFolder) } } // if iam_policy is not defined or excluded, don't generate it if object.IamPolicy == nil || object.IamPolicy.Exclude { return } tgc.GenerateIamPolicy(object, *templateData, outputFolder, generateCode, generateDocs) } func (tgc TerraformGoogleConversion) GenerateResource(object api.Resource, templateData TemplateData, outputFolder string, generateCode, generateDocs bool) { productName := tgc.Product.ApiName targetFolder := path.Join(outputFolder, "converters/google/resources/services", productName) if err := os.MkdirAll(targetFolder, os.ModePerm); err != nil { log.Println(fmt.Errorf("error creating parent directory %v: %v", targetFolder, err)) } targetFilePath := path.Join(targetFolder, fmt.Sprintf("%s_%s.go", productName, google.Underscore(object.Name))) templateData.GenerateTGCResourceFile(targetFilePath, object) } // Generate the IAM policy for this object. This is used to query and test // IAM policies separately from the resource itself // Docs are generated for the terraform provider, not here. func (tgc TerraformGoogleConversion) GenerateIamPolicy(object api.Resource, templateData TemplateData, outputFolder string, generateCode, generateDocs bool) { if !generateCode || object.IamPolicy.ExcludeTgc { return } productName := tgc.Product.ApiName targetFolder := path.Join(outputFolder, "converters/google/resources/services", productName) if err := os.MkdirAll(targetFolder, os.ModePerm); err != nil { log.Println(fmt.Errorf("error creating parent directory %v: %v", targetFolder, err)) } name := object.FilenameOverride if name == "" { name = google.Underscore(object.Name) } targetFilePath := path.Join(targetFolder, fmt.Sprintf("%s_%s_iam.go", productName, name)) templateData.GenerateTGCIamResourceFile(targetFilePath, object) targetFilePath = path.Join(targetFolder, fmt.Sprintf("iam_%s_%s.go", productName, name)) templateData.GenerateIamPolicyFile(targetFilePath, object) // Don't generate tests - we can rely on the terraform provider // to test these. } // Generates the list of resources func (tgc *TerraformGoogleConversion) generateCaiIamResources(products []*api.Product) { for _, productDefinition := range products { service := strings.ToLower(productDefinition.Name) for _, object := range productDefinition.Objects { if object.MinVersionObj().Name != "ga" || object.Exclude || object.ExcludeTgc { continue } var iamClassName string iamPolicy := object.IamPolicy if iamPolicy != nil && !iamPolicy.Exclude && !iamPolicy.ExcludeTgc { iamClassName = fmt.Sprintf("%s.ResourceConverter%s", service, object.ResourceName()) tgc.IamResources = append(tgc.IamResources, map[string]string{ "TerraformName": object.TerraformName(), "IamClassName": iamClassName, }) } } } } func (tgc TerraformGoogleConversion) CompileCommonFiles(outputFolder string, products []*api.Product, overridePath string) { log.Printf("Compiling common files for tgc.") tgc.generateCaiIamResources(products) tgc.NonDefinedTests = retrieveFullManifestOfNonDefinedTests() files := retrieveFullListOfTestFiles() for _, file := range files { tgc.Tests = append(tgc.Tests, strings.Split(file, ".")[0]) } tgc.Tests = slices.Compact(tgc.Tests) testSource := make(map[string]string) for target, source := range retrieveTestSourceCodeWithLocation(".tmpl") { target := strings.Replace(target, "go.tmpl", "go", 1) testSource[target] = source } templateData := NewTemplateData(outputFolder, tgc.TargetVersionName) tgc.CompileFileList(outputFolder, testSource, *templateData, products) resourceConverters := map[string]string{ "converters/google/resources/services/compute/compute_instance_helpers.go": "third_party/terraform/services/compute/compute_instance_helpers.go.tmpl", "converters/google/resources/resource_converters.go": "third_party/tgc/resource_converters.go.tmpl", "converters/google/resources/services/kms/iam_kms_key_ring.go": "third_party/terraform/services/kms/iam_kms_key_ring.go.tmpl", "converters/google/resources/services/kms/iam_kms_crypto_key.go": "third_party/terraform/services/kms/iam_kms_crypto_key.go.tmpl", "converters/google/resources/services/compute/metadata.go": "third_party/terraform/services/compute/metadata.go.tmpl", "converters/google/resources/services/compute/compute_instance.go": "third_party/tgc/services/compute/compute_instance.go.tmpl", } tgc.CompileFileList(outputFolder, resourceConverters, *templateData, products) } func (tgc TerraformGoogleConversion) CompileFileList(outputFolder string, files map[string]string, fileTemplate TemplateData, products []*api.Product) { if err := os.MkdirAll(outputFolder, os.ModePerm); err != nil { log.Println(fmt.Errorf("error creating output directory %v: %v", outputFolder, err)) } for target, source := range files { targetFile := filepath.Join(outputFolder, target) targetDir := filepath.Dir(targetFile) if err := os.MkdirAll(targetDir, os.ModePerm); err != nil { log.Println(fmt.Errorf("error creating output directory %v: %v", targetDir, err)) } templates := []string{ source, } formatFile := filepath.Ext(targetFile) == ".go" fileTemplate.GenerateFile(targetFile, source, tgc, formatFile, templates...) tgc.replaceImportPath(outputFolder, target) } } func retrieveFullManifestOfNonDefinedTests() []string { var tests []string fileMap := make(map[string]bool) files := retrieveFullListOfTestFiles() for _, file := range files { tests = append(tests, strings.Split(file, ".")[0]) fileMap[file] = true } tests = slices.Compact(tests) nonDefinedTests := google.Diff(tests, retrieveListOfManuallyDefinedTests()) nonDefinedTests = google.Reject(nonDefinedTests, func(file string) bool { return strings.HasSuffix(file, "_without_default_project") }) for _, test := range nonDefinedTests { _, ok := fileMap[fmt.Sprintf("%s.json", test)] if !ok { log.Fatalf("test file named %s.json expected but found none", test) } _, ok = fileMap[fmt.Sprintf("%s.tf", test)] if !ok { log.Fatalf("test file named %s.tf expected but found none", test) } } return nonDefinedTests } // Gets all of the test files in the folder third_party/tgc/tests/data func retrieveFullListOfTestFiles() []string { var testFiles []string files, err := ioutil.ReadDir("third_party/tgc/tests/data") if err != nil { log.Fatal(err) } for _, file := range files { testFiles = append(testFiles, file.Name()) } slices.Sort(testFiles) return testFiles } // Gets all of files in the folder third_party/tgc/tests/data func retrieveFullListOfTestTilesWithLocation() map[string]string { testFiles := make(map[string]string) files := retrieveFullListOfTestFiles() for _, file := range files { target := fmt.Sprintf("testdata/templates/%s", file) source := fmt.Sprintf("third_party/tgc/tests/data/%s", file) testFiles[target] = source } return testFiles } func retrieveTestSourceCodeWithLocation(suffix string) map[string]string { var fileNames []string path := "third_party/tgc/tests/source" files, err := ioutil.ReadDir(path) if err != nil { log.Fatal(err) } for _, file := range files { if filepath.Ext(file.Name()) == suffix { fileNames = append(fileNames, file.Name()) } } slices.Sort(fileNames) testSource := make(map[string]string) for _, file := range fileNames { target := fmt.Sprintf("test/%s", file) source := fmt.Sprintf("%s/%s", path, file) testSource[target] = source } return testSource } func retrieveListOfManuallyDefinedTests() []string { m1 := retrieveListOfManuallyDefinedTestsFromFile("third_party/tgc/tests/source/cli_test.go.tmpl") m2 := retrieveListOfManuallyDefinedTestsFromFile("third_party/tgc/tests/source/read_test.go.tmpl") return google.Concat(m1, m2) } // Reads the content of the file and then finds all of the tests in the contents func retrieveListOfManuallyDefinedTestsFromFile(file string) []string { data, err := os.ReadFile(file) if err != nil { log.Fatalf("Cannot open the file: %v", file) } var tests []string testsReg := regexp.MustCompile(`\s*name\s*:\s*"([^,]+)"`) matches := testsReg.FindAllStringSubmatch(string(data), -1) for _, testWithName := range matches { tests = append(tests, testWithName[1]) } return tests } func (tgc TerraformGoogleConversion) CopyCommonFiles(outputFolder string, generateCode, generateDocs bool) { log.Printf("Copying common files for tgc.") if !generateCode { return } tgc.CopyFileList(outputFolder, retrieveFullListOfTestTilesWithLocation()) tgc.CopyFileList(outputFolder, retrieveTestSourceCodeWithLocation(".go")) resourceConverters := map[string]string{ "converters/google/resources/cai/constants.go": "third_party/tgc/cai/constants.go", "converters/google/resources/constants.go": "third_party/tgc/constants.go", "converters/google/resources/cai.go": "third_party/tgc/cai.go", "converters/google/resources/cai/cai.go": "third_party/tgc/cai/cai.go", "converters/google/resources/cai/cai_test.go": "third_party/tgc/cai/cai_test.go", "converters/google/resources/services/resourcemanager/org_policy_policy.go": "third_party/tgc/services/resourcemanager/org_policy_policy.go", "converters/google/resources/getconfig.go": "third_party/tgc/getconfig.go", "converters/google/resources/services/resourcemanager/folder.go": "third_party/tgc/services/resourcemanager/folder.go", "converters/google/resources/getconfig_test.go": "third_party/tgc/getconfig_test.go", "converters/google/resources/cai/json_map.go": "third_party/tgc/cai/json_map.go", "converters/google/resources/cai/string_helpers.go": "third_party/tgc/cai/string_helpers.go", "converters/google/resources/services/resourcemanager/project.go": "third_party/tgc/services/resourcemanager/project.go", "converters/google/resources/services/sql/sql_database_instance.go": "third_party/tgc/services/sql/sql_database_instance.go", "converters/google/resources/services/storage/storage_bucket.go": "third_party/tgc/services/storage/storage_bucket.go", "converters/google/resources/services/cloudfunctions/cloudfunctions_function.go": "third_party/tgc/services/cloudfunctions/cloudfunctions_function.go", "converters/google/resources/services/cloudfunctions/cloudfunctions_cloud_function.go": "third_party/tgc/services/cloudfunctions/cloudfunctions_cloud_function.go", "converters/google/resources/services/bigquery/bigquery_table.go": "third_party/tgc/services/bigquery/bigquery_table.go", "converters/google/resources/services/bigtable/bigtable_cluster.go": "third_party/tgc/services/bigtable/bigtable_cluster.go", "converters/google/resources/services/bigtable/bigtable_instance.go": "third_party/tgc/services/bigtable/bigtable_instance.go", "converters/google/resources/cai/iam_helpers.go": "third_party/tgc/cai/iam_helpers.go", "converters/google/resources/cai/iam_helpers_test.go": "third_party/tgc/cai/iam_helpers_test.go", "converters/google/resources/services/resourcemanager/organization_iam.go": "third_party/tgc/services/resourcemanager/organization_iam.go", "converters/google/resources/services/resourcemanager/project_iam.go": "third_party/tgc/services/resourcemanager/project_iam.go", "converters/google/resources/services/resourcemanager/project_organization_policy.go": "third_party/tgc/services/resourcemanager/project_organization_policy.go", "converters/google/resources/services/resourcemanager/folder_organization_policy.go": "third_party/tgc/services/resourcemanager/folder_organization_policy.go", "converters/google/resources/services/resourcemanager/folder_iam.go": "third_party/tgc/services/resourcemanager/folder_iam.go", "converters/google/resources/services/container/container.go": "third_party/tgc/services/container/container.go", "converters/google/resources/services/resourcemanager/project_service.go": "third_party/tgc/services/resourcemanager/project_service.go", "converters/google/resources/services/monitoring/monitoring_slo_helper.go": "third_party/tgc/services/monitoring/monitoring_slo_helper.go", "converters/google/resources/services/resourcemanager/service_account.go": "third_party/tgc/services/resourcemanager/service_account.go", "converters/google/resources/services/compute/image.go": "third_party/terraform/services/compute/image.go", "converters/google/resources/services/compute/disk_type.go": "third_party/terraform/services/compute/disk_type.go", "converters/google/resources/services/kms/kms_utils.go": "third_party/terraform/services/kms/kms_utils.go", "converters/google/resources/services/sourcerepo/source_repo_utils.go": "third_party/terraform/services/sourcerepo/source_repo_utils.go", "converters/google/resources/services/pubsub/pubsub_utils.go": "third_party/terraform/services/pubsub/pubsub_utils.go", "converters/google/resources/services/resourcemanager/iam_organization.go": "third_party/terraform/services/resourcemanager/iam_organization.go", "converters/google/resources/services/resourcemanager/iam_folder.go": "third_party/terraform/services/resourcemanager/iam_folder.go", "converters/google/resources/services/resourcemanager/iam_project.go": "third_party/terraform/services/resourcemanager/iam_project.go", "converters/google/resources/services/privateca/privateca_utils.go": "third_party/terraform/services/privateca/privateca_utils.go", "converters/google/resources/services/bigquery/iam_bigquery_dataset.go": "third_party/terraform/services/bigquery/iam_bigquery_dataset.go", "converters/google/resources/services/bigquery/bigquery_dataset_iam.go": "third_party/tgc/services/bigquery/bigquery_dataset_iam.go", "converters/google/resources/services/compute/compute_security_policy.go": "third_party/tgc/services/compute/compute_security_policy.go", "converters/google/resources/services/eventarc/eventarc_utils.go": "third_party/terraform/services/eventarc/eventarc_utils.go", "converters/google/resources/services/kms/kms_key_ring_iam.go": "third_party/tgc/services/kms/kms_key_ring_iam.go", "converters/google/resources/services/kms/kms_crypto_key_iam.go": "third_party/tgc/services/kms/kms_crypto_key_iam.go", "converters/google/resources/services/resourcemanager/project_iam_custom_role.go": "third_party/tgc/services/resourcemanager/project_iam_custom_role.go", "converters/google/resources/services/resourcemanager/organization_iam_custom_role.go": "third_party/tgc/services/resourcemanager/organization_iam_custom_role.go", "converters/google/resources/services/pubsub/iam_pubsub_subscription.go": "third_party/terraform/services/pubsub/iam_pubsub_subscription.go", "converters/google/resources/services/pubsub/pubsub_subscription_iam.go": "third_party/tgc/services/pubsub/pubsub_subscription_iam.go", "converters/google/resources/services/spanner/iam_spanner_database.go": "third_party/terraform/services/spanner/iam_spanner_database.go", "converters/google/resources/services/spanner/spanner_database_iam.go": "third_party/tgc/services/spanner/spanner_database_iam.go", "converters/google/resources/services/spanner/iam_spanner_instance.go": "third_party/terraform/services/spanner/iam_spanner_instance.go", "converters/google/resources/services/spanner/spanner_instance_iam.go": "third_party/tgc/services/spanner/spanner_instance_iam.go", "converters/google/resources/services/storage/storage_bucket_iam.go": "third_party/tgc/services/storage/storage_bucket_iam.go", "converters/google/resources/services/resourcemanager/organization_policy.go": "third_party/tgc/services/resourcemanager/organization_policy.go", "converters/google/resources/services/storage/iam_storage_bucket.go": "third_party/tgc/services/storage/iam_storage_bucket.go", "ancestrymanager/ancestrymanager.go": "third_party/tgc/ancestrymanager/ancestrymanager.go", "ancestrymanager/ancestrymanager_test.go": "third_party/tgc/ancestrymanager/ancestrymanager_test.go", "ancestrymanager/ancestryutil.go": "third_party/tgc/ancestrymanager/ancestryutil.go", "ancestrymanager/ancestryutil_test.go": "third_party/tgc/ancestrymanager/ancestryutil_test.go", "converters/google/convert.go": "third_party/tgc/convert.go", "converters/google/convert_test.go": "third_party/tgc/convert_test.go", "tfdata/fake_resource_data.go": "third_party/tgc/tfdata/fake_resource_data.go", "tfdata/fake_resource_data_test.go": "third_party/tgc/tfdata/fake_resource_data_test.go", "converters/google/resources/services/compute/compute_instance_group.go": "third_party/tgc/services/compute/compute_instance_group.go", "converters/google/resources/services/dataflow/job.go": "third_party/tgc/services/dataflow/job.go", "converters/google/resources/services/resourcemanager/service_account_key.go": "third_party/tgc/services/resourcemanager/service_account_key.go", "converters/google/resources/services/compute/compute_target_pool.go": "third_party/tgc/services/compute/compute_target_pool.go", "converters/google/resources/services/dataproc/dataproc_cluster.go": "third_party/tgc/services/dataproc/dataproc_cluster.go", "converters/google/resources/services/composer/composer_environment.go": "third_party/tgc/services/composer/composer_environment.go", "converters/google/resources/services/compute/commitment.go": "third_party/tgc/services/compute/commitment.go", "converters/google/resources/services/firebase/firebase_project.go": "third_party/tgc/services/firebase/firebase_project.go", "converters/google/resources/services/appengine/appengine_application.go": "third_party/tgc/services/appengine/appengine_application.go", "converters/google/resources/services/apikeys/apikeys_key.go": "third_party/tgc/services/apikeys/apikeys_key.go", "converters/google/resources/services/logging/logging_folder_bucket_config.go": "third_party/tgc/services/logging/logging_folder_bucket_config.go", "converters/google/resources/services/logging/logging_organization_bucket_config.go": "third_party/tgc/services/logging/logging_organization_bucket_config.go", "converters/google/resources/services/logging/logging_project_bucket_config.go": "third_party/tgc/services/logging/logging_project_bucket_config.go", "converters/google/resources/services/logging/logging_billing_account_bucket_config.go": "third_party/tgc/services/logging/logging_billing_account_bucket_config.go", "converters/google/resources/services/appengine/appengine_standard_version.go": "third_party/tgc/services/appengine/appengine_standard_version.go", } tgc.CopyFileList(outputFolder, resourceConverters) } func (tgc TerraformGoogleConversion) CopyFileList(outputFolder string, files map[string]string) { for target, source := range files { targetFile := filepath.Join(outputFolder, target) targetDir := filepath.Dir(targetFile) if err := os.MkdirAll(targetDir, os.ModePerm); err != nil { log.Println(fmt.Errorf("error creating output directory %v: %v", targetDir, err)) } // If we've modified a file since starting an MM run, it's a reasonable // assumption that it was this run that modified it. if info, err := os.Stat(targetFile); !errors.Is(err, os.ErrNotExist) && tgc.StartTime.Before(info.ModTime()) { log.Fatalf("%s was already modified during this run at %s", targetFile, info.ModTime().String()) } sourceByte, err := os.ReadFile(source) if err != nil { log.Fatalf("Cannot read source file %s while copying: %s", source, err) } err = os.WriteFile(targetFile, sourceByte, 0644) if err != nil { log.Fatalf("Cannot write target file %s while copying: %s", target, err) } // Replace import path based on version (beta/alpha) if filepath.Ext(target) == ".go" || filepath.Ext(target) == ".mod" { tgc.replaceImportPath(outputFolder, target) } } } func (tgc TerraformGoogleConversion) replaceImportPath(outputFolder, target string) { // Replace import paths to reference the resources dir instead of the google provider targetFile := filepath.Join(outputFolder, target) sourceByte, err := os.ReadFile(targetFile) if err != nil { log.Fatalf("Cannot read file %s to replace import path: %s", targetFile, err) } // replace google to google-beta gaImportPath := ImportPathFromVersion("ga") sourceByte = bytes.Replace(sourceByte, []byte(gaImportPath), []byte(TERRAFORM_PROVIDER_BETA+"/"+RESOURCE_DIRECTORY_BETA), -1) err = os.WriteFile(targetFile, sourceByte, 0644) if err != nil { log.Fatalf("Cannot write file %s to replace import path: %s", target, err) } }