func resourceAlicloudEmrV2ClusterRead()

in alicloud/resource_alicloud_emrv2_cluster.go [1311:2007]


func resourceAlicloudEmrV2ClusterRead(d *schema.ResourceData, meta interface{}) error {
	client := meta.(*connectivity.AliyunClient)
	emrService := EmrService{client}

	object, err := emrService.GetEmrV2Cluster(d.Id())

	if err != nil {
		if NotFoundError(err) {
			d.SetId("")
			return nil
		}
		return WrapError(err)
	}

	d.Set("cluster_name", object["ClusterName"])
	d.Set("cluster_type", object["ClusterType"])
	d.Set("payment_type", object["PaymentType"])
	d.Set("release_version", object["ReleaseVersion"])
	d.Set("deploy_mode", object["DeployMode"])
	d.Set("security_mode", object["SecurityMode"])
	d.Set("resource_group_id", object["ResourceGroupId"])
	d.Set("log_collect_strategy", object["LogCollectStrategy"])
	d.Set("deletion_protection", object["DeletionProtection"])

	if _, ok := object["SubscriptionConfig"]; ok {
		sc := d.Get("subscription_config").(*schema.Set).List()
		if len(sc) > 0 {
			d.Set("subscription_config", []map[string]interface{}{sc[0].(map[string]interface{})})
		}
	}

	var nodeAttributes []map[string]interface{}
	if v, ok := object["NodeAttributes"]; ok {
		nodeAttributesMap := v.(map[string]interface{})
		nodeAttribute := map[string]interface{}{
			"vpc_id":                nodeAttributesMap["VpcId"],
			"zone_id":               nodeAttributesMap["ZoneId"],
			"security_group_id":     nodeAttributesMap["SecurityGroupId"],
			"ram_role":              nodeAttributesMap["RamRole"],
			"key_pair_name":         nodeAttributesMap["KeyPairName"],
			"data_disk_encrypted":   nodeAttributesMap["DataDiskEncrypted"],
			"system_disk_encrypted": nodeAttributesMap["SystemDiskEncrypted"],
		}
		if v, exists := nodeAttributesMap["DataDiskKMSKeyId"]; exists && v.(string) != "" {
			nodeAttribute["data_disk_kms_key_id"] = v
		}
		if v, exists := nodeAttributesMap["SystemDiskKMSKeyId"]; exists && v.(string) != "" {
			nodeAttribute["system_disk_kms_key_id"] = v
		}

		oldNodeAttributes := d.Get("node_attributes")
		if oldNodeAttributes != nil && oldNodeAttributes.(*schema.Set).Len() > 0 {
			oldNodeAttributesMap := d.Get("node_attributes").(*schema.Set).List()[0].(map[string]interface{})
			if _, exists := oldNodeAttributesMap["system_disk_encrypted"]; exists {
				nodeAttribute["system_disk_encrypted"] = nodeAttributesMap["SystemDiskEncrypted"]
			}
			if value, exists := oldNodeAttributesMap["system_disk_kms_key_id"]; exists && value != "" {
				nodeAttribute["system_disk_kms_key_id"] = nodeAttributesMap["SystemDiskKMSKeyId"]
			}
		}

		nodeAttributes = append(nodeAttributes, nodeAttribute)
		d.Set("node_attributes", nodeAttributes)
	}
	var response map[string]interface{}
	action := "ListApplications"
	request := map[string]interface{}{
		"RegionId":  client.RegionId,
		"ClusterId": d.Id(),
	}
	wait := incrementalWait(3*time.Second, 3*time.Second)
	err = resource.Retry(5*time.Minute, func() *resource.RetryError {
		response, err = client.RpcPost("Emr", "2021-03-20", action, nil, request, true)
		if err != nil {
			if NeedRetry(err) {
				wait()
				return resource.RetryableError(err)
			}
			return resource.NonRetryableError(err)
		}
		return nil
	})
	addDebug(action, response, request)
	if err != nil {
		return WrapErrorf(err, DefaultErrorMsg, d.Id(), action, AlibabaCloudSdkGoERROR)
	}
	v, err := jsonpath.Get("$.Applications", response)
	if err != nil {
		return WrapErrorf(err, FailedGetAttributeMsg, d.Id(), "$.Applications", response)
	}
	if v != nil && len(v.([]interface{})) > 0 {
		emrInternalApp := map[string]struct{}{
			"TRE-EXTENSION":        {},
			"KERBEROS":             {},
			"METASTORE":            {},
			"MYSQL":                {},
			"SPARK-EXTENSION":      {},
			"SPARK-NATIVE":         {},
			"TAIHAODOCTOR":         {},
			"EMRHOOK":              {},
			"JINDOSDK":             {},
			"TRE":                  {},
			"EMRRUNTIME":           {},
			"EMRRUNTIME-EXTENSION": {},
		}
		var applications []string
		for _, item := range v.([]interface{}) {
			app := strings.ToUpper(item.(map[string]interface{})["ApplicationName"].(string))
			if _, ok := emrInternalApp[app]; ok {
				continue
			}
			applications = append(applications, app)
		}
		d.Set("applications", applications)
	}

	if v, ok := d.GetOk("application_configs"); ok && len(v.(*schema.Set).List()) > 0 {
		var applicationConfigs []map[string]interface{}
		for _, ac := range v.(*schema.Set).List() {
			applicationConfigs = append(applicationConfigs, ac.(map[string]interface{}))
		}
		d.Set("application_configs", applicationConfigs)
	}

	action = "ListScripts"
	listScriptsRequest := map[string]interface{}{
		"RegionId":   client.RegionId,
		"ClusterId":  d.Id(),
		"ScriptType": "BOOTSTRAP",
	}
	err = resource.Retry(5*time.Minute, func() *resource.RetryError {
		response, err = client.RpcPost("Emr", "2021-03-20", action, nil, listScriptsRequest, true)
		if err != nil {
			if NeedRetry(err) {
				wait()
				return resource.RetryableError(err)
			}
			return resource.NonRetryableError(err)
		}
		return nil
	})
	addDebug(action, response, request)
	if err != nil {
		return WrapErrorf(err, DefaultErrorMsg, d.Id(), action, AlibabaCloudSdkGoERROR)
	}
	v, err = jsonpath.Get("$.Scripts", response)
	if err != nil {
		return WrapErrorf(err, FailedGetAttributeMsg, d.Id(), "$.Scripts", response)
	}
	if v != nil && len(v.([]interface{})) > 0 {
		scriptsMaps := v.([]interface{})
		var scripts []map[string]interface{}
		for _, item := range scriptsMaps {
			scriptMap := item.(map[string]interface{})
			script := map[string]interface{}{
				"script_name":             scriptMap["ScriptName"],
				"script_path":             scriptMap["ScriptPath"],
				"script_args":             scriptMap["ScriptArgs"],
				"execution_moment":        scriptMap["ExecutionMoment"],
				"execution_fail_strategy": scriptMap["ExecutionFailStrategy"],
			}

			if v, ok := scriptMap["NodeSelector"]; ok && len(v.(map[string]interface{})) > 0 {
				nodeSelectorMap := v.(map[string]interface{})
				nodeSelector := map[string]interface{}{
					"node_select_type": nodeSelectorMap["NodeSelectType"],
				}

				if nodeGroupId, exists := nodeSelectorMap["NodeGroupId"]; exists && nodeGroupId.(string) != "" {
					nodeSelector["node_group_id"] = nodeGroupId
				}

				if ngIDs, exists := nodeSelectorMap["NodeGroupIds"]; exists && len(ngIDs.([]interface{})) > 0 {
					var nodeGroupIDs []string
					for _, ngID := range ngIDs.([]interface{}) {
						nodeGroupIDs = append(nodeGroupIDs, ngID.(string))
					}
					nodeSelector["node_group_ids"] = nodeGroupIDs
				}

				if nodeGroupName, exists := nodeSelectorMap["NodeGroupName"]; exists && nodeGroupName.(string) != "" {
					nodeSelector["node_group_name"] = nodeGroupName
				}

				if ngNames, exists := nodeSelectorMap["NodeGroupNames"]; exists && len(ngNames.([]interface{})) > 0 {
					var nodeGroupNames []string
					for _, ngName := range ngNames.([]interface{}) {
						nodeGroupNames = append(nodeGroupNames, ngName.(string))
					}
					nodeSelector["node_group_names"] = nodeGroupNames
				}

				if ngTypes, exists := nodeSelectorMap["NodeGroupTypes"]; exists && len(ngTypes.([]interface{})) > 0 {
					var nodeGroupTypes []string
					for _, ngType := range ngTypes.([]interface{}) {
						nodeGroupTypes = append(nodeGroupTypes, ngType.(string))
					}
					nodeSelector["node_group_types"] = nodeGroupTypes
				}

				if nn, exists := nodeSelectorMap["NodeNames"]; exists && len(nn.([]interface{})) > 0 {
					var nodeNames []string
					for _, nodeName := range nn.([]interface{}) {
						nodeNames = append(nodeNames, nodeName.(string))
					}
					nodeSelector["node_names"] = nodeNames
				}
				script["node_selector"] = []map[string]interface{}{nodeSelector}
			}
			scripts = append(scripts, script)
		}
		d.Set("bootstrap_scripts", scripts)
	}

	action = "ListNodeGroups"
	request["MaxResults"] = PageSizeLarge
	request["NodeGroupStates"] = []string{"RUNNING"}
	var nodeGroupObjects []interface{}

	for {
		err = resource.Retry(5*time.Minute, func() *resource.RetryError {
			response, err = client.RpcPost("Emr", "2021-03-20", action, nil, request, true)
			if err != nil {
				if NeedRetry(err) {
					wait()
					return resource.RetryableError(err)
				}
				return resource.NonRetryableError(err)
			}
			return nil
		})
		addDebug(action, response, request)
		if err != nil {
			return WrapErrorf(err, DefaultErrorMsg, d.Id(), action, AlibabaCloudSdkGoERROR)
		}

		nodeGroupResp, err := jsonpath.Get("$.NodeGroups", response)
		if err != nil {
			return WrapErrorf(err, FailedGetAttributeMsg, d.Id(), "$.NodeGroups", response)
		}
		if nodeGroupResp != nil && len(nodeGroupResp.([]interface{})) > 0 {
			for _, ngr := range nodeGroupResp.([]interface{}) {
				nodeGroupObjects = append(nodeGroupObjects, ngr)
			}
			_, nextTokenExists := response["NextToken"]
			if len(nodeGroupResp.([]interface{})) < PageSizeLarge {
				break
			} else if len(nodeGroupResp.([]interface{})) == PageSizeLarge && !nextTokenExists {
				break
			}
		} else {
			break
		}

		nextToken, err := jsonpath.Get("$.NextToken", response)
		if err != nil {
			return WrapErrorf(err, FailedGetAttributeMsg, action, "$.NextToken", response)
		}
		request["NextToken"] = nextToken
	}
	if len(nodeGroupObjects) > 0 {
		v = nodeGroupObjects
	}
	delete(request, "NextToken")

	if v != nil && len(v.([]interface{})) > 0 {
		oldNodeGroupsMap := map[string]map[string]interface{}{}
		indexNodeGroupMap := map[string]int{}
		if oldNodeGroups, ok := d.GetOk("node_groups"); ok {
			for index, item := range oldNodeGroups.([]interface{}) {
				oldNodeGroup := item.(map[string]interface{})
				oldNodeGroupsMap[oldNodeGroup["node_group_name"].(string)] = oldNodeGroup
				indexNodeGroupMap[oldNodeGroup["node_group_name"].(string)] = index
			}
		}

		nodeGroupMaps := v.([]interface{})
		var nodeGroups []map[string]interface{}
		for _, item := range nodeGroupMaps {
			nodeGroupMap := item.(map[string]interface{})
			nodeGroup := map[string]interface{}{
				"node_group_type":      nodeGroupMap["NodeGroupType"],
				"node_group_name":      nodeGroupMap["NodeGroupName"],
				"payment_type":         nodeGroupMap["PaymentType"],
				"with_public_ip":       nodeGroupMap["WithPublicIp"],
				"graceful_shutdown":    nodeGroupMap["GracefulShutdown"],
				"spot_instance_remedy": nodeGroupMap["SpotInstanceRemedy"],
				"node_count":           nodeGroupMap["RunningNodeCount"],
			}

			if oldNodeGroup, exists := oldNodeGroupsMap[nodeGroupMap["NodeGroupName"].(string)]; exists {
				if v, ok := oldNodeGroup["subscription_config"]; ok {
					var subscriptionConfig []map[string]interface{}
					for _, item := range v.(*schema.Set).List() {
						subscriptionConfig = append(subscriptionConfig, item.(map[string]interface{}))
					}
					nodeGroup["subscription_config"] = subscriptionConfig
				}

				if v, ok := oldNodeGroup["spot_bid_prices"]; ok {
					var spotBidPrices []map[string]interface{}
					for _, item := range v.(*schema.Set).List() {
						spotBidPrices = append(spotBidPrices, item.(map[string]interface{}))
					}
					nodeGroup["spot_bid_prices"] = spotBidPrices
				}

				if v, ok := oldNodeGroup["auto_scaling_policy"]; ok && len(v.([]interface{})) == 1 {
					var scalingPolicies []map[string]interface{}
					scalingPolicy := map[string]interface{}{}
					scalingPolicyMap := v.([]interface{})[0].(map[string]interface{})
					if scalingPolicyValue, scalingPolicyExists := scalingPolicyMap["scaling_rules"]; scalingPolicyExists && len(scalingPolicyValue.([]interface{})) > 0 {
						var scalingRules []map[string]interface{}
						for _, sr := range scalingPolicyValue.([]interface{}) {
							scalingRule := map[string]interface{}{}
							scalingRuleMap := sr.(map[string]interface{})
							if scalingRuleValue, scalingRuleExists := scalingRuleMap["rule_name"]; scalingRuleExists && scalingRuleValue.(string) != "" {
								scalingRule["rule_name"] = scalingRuleValue
							}
							if scalingRuleValue, scalingRuleExists := scalingRuleMap["trigger_type"]; scalingRuleExists && scalingRuleValue.(string) != "" {
								scalingRule["trigger_type"] = scalingRuleValue
							}
							if scalingRuleValue, scalingRuleExists := scalingRuleMap["activity_type"]; scalingRuleExists && scalingRuleValue.(string) != "" {
								scalingRule["activity_type"] = scalingRuleValue
							}
							if scalingRuleValue, scalingRuleExists := scalingRuleMap["adjustment_type"]; scalingRuleExists && scalingRuleValue.(string) != "" {
								scalingRule["adjustment_type"] = scalingRuleValue
							}
							if scalingRuleValue, scalingRuleExists := scalingRuleMap["adjustment_value"]; scalingRuleExists && scalingRuleValue.(int) != 0 {
								scalingRule["adjustment_value"] = scalingRuleValue
							}
							if scalingRuleValue, scalingRuleExists := scalingRuleMap["min_adjustment_value"]; scalingRuleExists {
								scalingRule["min_adjustment_value"] = scalingRuleValue
							}
							if scalingRuleValue, scalingRuleExists := scalingRuleMap["time_trigger"]; scalingRuleExists && len(scalingRuleValue.([]interface{})) > 0 {
								var timeTriggers []map[string]interface{}
								for _, tt := range scalingRuleValue.([]interface{}) {
									timeTrigger := map[string]interface{}{}
									timeTriggerMap := tt.(map[string]interface{})
									if timeTriggerValue, timeTriggerExists := timeTriggerMap["launch_time"]; timeTriggerExists && timeTriggerValue.(string) != "" {
										timeTrigger["launch_time"] = timeTriggerValue
									}
									if timeTriggerValue, timeTriggerExists := timeTriggerMap["start_time"]; timeTriggerExists && timeTriggerValue.(string) != "" {
										timeTrigger["start_time"] = timeTriggerValue
									}
									if timeTriggerValue, timeTriggerExists := timeTriggerMap["end_time"]; timeTriggerExists && timeTriggerValue.(string) != "" {
										timeTrigger["end_time"] = timeTriggerValue
									}
									if timeTriggerValue, timeTriggerExists := timeTriggerMap["launch_expiration_time"]; timeTriggerExists && timeTriggerValue.(int) != 0 {
										timeTrigger["launch_expiration_time"] = timeTriggerValue
									}
									if timeTriggerValue, timeTriggerExists := timeTriggerMap["recurrence_type"]; timeTriggerExists && timeTriggerValue.(string) != "" {
										timeTrigger["recurrence_type"] = timeTriggerValue
									}
									if timeTriggerValue, timeTriggerExists := timeTriggerMap["recurrence_value"]; timeTriggerExists && timeTriggerValue.(string) != "" {
										timeTrigger["recurrence_value"] = timeTriggerValue
									}
									timeTriggers = append(timeTriggers, timeTrigger)
								}
								scalingRule["time_trigger"] = timeTriggers
							}
							if scalingRuleValue, scalingRuleExists := scalingRuleMap["metrics_trigger"]; scalingRuleExists && len(scalingRuleValue.([]interface{})) > 0 {
								var metricsTriggers []map[string]interface{}
								for _, mt := range scalingRuleValue.([]interface{}) {
									metricsTrigger := map[string]interface{}{}
									metricsTriggerMap := mt.(map[string]interface{})
									if metricsTriggerValue, metricsTriggerExists := metricsTriggerMap["time_window"]; metricsTriggerExists && metricsTriggerValue.(int) != 0 {
										metricsTrigger["time_window"] = metricsTriggerValue
									}
									if metricsTriggerValue, metricsTriggerExists := metricsTriggerMap["evaluation_count"]; metricsTriggerExists && metricsTriggerValue.(int) != 0 {
										metricsTrigger["evaluation_count"] = metricsTriggerValue
									}
									if metricsTriggerValue, metricsTriggerExists := metricsTriggerMap["cool_down_interval"]; metricsTriggerExists && metricsTriggerValue.(int) != 0 {
										metricsTrigger["cool_down_interval"] = metricsTriggerValue
									}
									if metricsTriggerValue, metricsTriggerExists := metricsTriggerMap["condition_logic_operator"]; metricsTriggerExists && metricsTriggerValue.(string) != "" {
										metricsTrigger["condition_logic_operator"] = metricsTriggerValue
									}
									if metricsTriggerValue, metricsTriggerExists := metricsTriggerMap["time_constraints"]; metricsTriggerExists && len(metricsTriggerValue.([]interface{})) > 0 {
										var timeConstraints []map[string]interface{}
										for _, tc := range metricsTriggerValue.([]interface{}) {
											timeConstraint := map[string]interface{}{}
											timeConstraintMap := tc.(map[string]interface{})
											if timeConstraintValue, timeConstraintExists := timeConstraintMap["start_time"]; timeConstraintExists && timeConstraintValue.(string) != "" {
												timeConstraint["start_time"] = timeConstraintValue
											}
											if timeConstraintValue, timeConstraintExists := timeConstraintMap["end_time"]; timeConstraintExists && timeConstraintValue.(string) != "" {
												timeConstraint["end_time"] = timeConstraintValue
											}
											timeConstraints = append(timeConstraints, timeConstraint)
										}
										metricsTrigger["time_constraints"] = timeConstraints
									}
									if metricsTriggerValue, metricsTriggerExists := metricsTriggerMap["conditions"]; metricsTriggerExists && len(metricsTriggerValue.([]interface{})) > 0 {
										var conditions []map[string]interface{}
										for _, cd := range metricsTriggerValue.([]interface{}) {
											condition := map[string]interface{}{}
											conditionMap := cd.(map[string]interface{})
											if conditionValue, conditionExists := conditionMap["metric_name"]; conditionExists && conditionValue.(string) != "" {
												condition["metric_name"] = conditionValue
											}
											if conditionValue, conditionExists := conditionMap["statistics"]; conditionExists && conditionValue.(string) != "" {
												condition["statistics"] = conditionValue
											}
											if conditionValue, conditionExists := conditionMap["comparison_operator"]; conditionExists && conditionValue.(string) != "" {
												condition["comparison_operator"] = conditionValue
											}
											if conditionValue, conditionExists := conditionMap["threshold"]; conditionExists && conditionValue.(float64) != 0.0 {
												condition["threshold"] = conditionValue
											}
											if conditionValue, conditionExists := conditionMap["tags"]; conditionExists && len(conditionValue.([]interface{})) > 0 {
												var tags []map[string]interface{}
												for _, tg := range conditionValue.([]interface{}) {
													tag := map[string]interface{}{}
													tagMap := tg.(map[string]interface{})
													if tagValue, tagExists := tagMap["key"]; tagExists && tagValue.(string) != "" {
														tag["key"] = tagValue
													}
													if tagValue, tagExists := tagMap["value"]; tagExists && tagValue.(string) != "" {
														tag["value"] = tagValue
													}
													tags = append(tags, tag)
												}
												condition["tags"] = tags
											}
											conditions = append(conditions, condition)
										}
										metricsTrigger["conditions"] = conditions
									}
									metricsTriggers = append(metricsTriggers, metricsTrigger)
								}
								scalingRule["metrics_trigger"] = metricsTriggers
							}
							scalingRules = append(scalingRules, scalingRule)
						}
						scalingPolicy["scaling_rules"] = scalingRules
					}
					if scalingPolicyValue, scalingPolicyExists := scalingPolicyMap["constraints"]; scalingPolicyExists && len(scalingPolicyValue.([]interface{})) > 0 {
						var constraints []map[string]interface{}
						for _, ct := range scalingPolicyValue.([]interface{}) {
							constraint := map[string]interface{}{}
							constraintMap := ct.(map[string]interface{})
							if constraintValue, constraintExists := constraintMap["max_capacity"]; constraintExists && constraintValue.(int) != 0 {
								constraint["max_capacity"] = constraintValue
							}
							if constraintValue, constraintExists := constraintMap["min_capacity"]; constraintExists && constraintValue.(int) != 0 {
								constraint["min_capacity"] = constraintValue
							}
							constraints = append(constraints, constraint)
						}
						scalingPolicy["constraints"] = constraints
					}
					scalingPolicies = append(scalingPolicies, scalingPolicy)
					nodeGroup["auto_scaling_policy"] = scalingPolicies
				}
			}

			if v, ok := nodeGroupMap["VSwitchIds"]; ok && len(v.([]interface{})) > 0 {
				var vSwitchIDs []string
				for _, item := range v.([]interface{}) {
					vSwitchIDs = append(vSwitchIDs, item.(string))
				}
				nodeGroup["vswitch_ids"] = vSwitchIDs
			}

			if v, ok := nodeGroupMap["AdditionalSecurityGroupIds"]; ok && len(v.([]interface{})) > 0 {
				var additionalSecurityGroupIDs []string
				for _, item := range v.([]interface{}) {
					additionalSecurityGroupIDs = append(additionalSecurityGroupIDs, item.(string))
				}
				nodeGroup["additional_security_group_ids"] = additionalSecurityGroupIDs
			}

			if v, ok := nodeGroupMap["InstanceTypes"]; ok && len(v.([]interface{})) > 0 {
				var instanceTypes []string
				for _, item := range v.([]interface{}) {
					instanceTypes = append(instanceTypes, item.(string))
				}
				nodeGroup["instance_types"] = instanceTypes
			}

			if v, ok := nodeGroupMap["AckConfig"]; ok && len(v.(map[string]interface{})) > 0 {
				var ackConfigs []map[string]interface{}
				ackConfig := map[string]interface{}{}
				m := v.(map[string]interface{})
				ackConfig["ack_instance_id"] = m["AckInstanceId"]
				if value, exists := m["NodeSelectors"]; exists && len(value.([]interface{})) > 0 {
					var nodeSelectors []map[string]interface{}
					for _, ackConfigValue := range value.([]interface{}) {
						nodeSelectors = append(nodeSelectors, map[string]interface{}{
							"key":   ackConfigValue.(map[string]interface{})["Key"],
							"value": ackConfigValue.(map[string]interface{})["Value"],
						})
					}
					ackConfig["node_selectors"] = nodeSelectors
				}
				if value, exists := m["Tolerations"]; exists && len(value.([]interface{})) > 0 {
					var tolerations []map[string]interface{}
					for _, ackConfigValue := range value.([]interface{}) {
						tolerations = append(tolerations, map[string]interface{}{
							"key":      ackConfigValue.(map[string]interface{})["Key"],
							"value":    ackConfigValue.(map[string]interface{})["Value"],
							"operator": ackConfigValue.(map[string]interface{})["Operator"],
							"effect":   ackConfigValue.(map[string]interface{})["Effect"],
						})
					}
					ackConfig["tolerations"] = tolerations
				}
				if value, exists := m["Namespace"]; exists && value.(string) != "" {
					ackConfig["namespace"] = value
				}
				if value, exists := m["RequestCpu"]; exists {
					ackConfig["request_cpu"] = value
				}
				if value, exists := m["RequestMemory"]; exists {
					ackConfig["request_memory"] = value
				}
				if value, exists := m["LimitCpu"]; exists {
					ackConfig["limit_cpu"] = value
				}
				if value, exists := m["LimitMemory"]; exists {
					ackConfig["limit_memory"] = value
				}
				if value, exists := m["CustomLabels"]; exists && len(value.([]interface{})) > 0 {
					var customLabels []map[string]interface{}
					for _, ackConfigValue := range value.([]interface{}) {
						customLabels = append(customLabels, map[string]interface{}{
							"key":   ackConfigValue.(map[string]interface{})["Key"],
							"value": ackConfigValue.(map[string]interface{})["Value"],
						})
					}
					ackConfig["custom_labels"] = customLabels
				}
				if value, exists := m["CustomAnnotations"]; exists && len(value.([]interface{})) > 0 {
					var customAnnotations []map[string]interface{}
					for _, ackConfigValue := range value.([]interface{}) {
						customAnnotations = append(customAnnotations, map[string]interface{}{
							"key":   ackConfigValue.(map[string]interface{})["Key"],
							"value": ackConfigValue.(map[string]interface{})["Value"],
						})
					}
					ackConfig["custom_annotations"] = customAnnotations
				}
				if value, exists := m["Pvcs"]; exists && len(value.([]interface{})) > 0 {
					var pvcs []map[string]interface{}
					for _, ackConfigValue := range value.([]interface{}) {
						pvcs = append(pvcs, map[string]interface{}{
							"name":                    ackConfigValue.(map[string]interface{})["Name"],
							"path":                    ackConfigValue.(map[string]interface{})["Path"],
							"data_disk_storage_class": ackConfigValue.(map[string]interface{})["DataDiskStorageClass"],
							"data_disk_size":          ackConfigValue.(map[string]interface{})["DataDiskSize"],
						})
					}
					ackConfig["pvcs"] = pvcs
				}
				if value, exists := m["Volumes"]; exists && len(value.([]interface{})) > 0 {
					var volumes []map[string]interface{}
					for _, ackConfigValue := range value.([]interface{}) {
						volumes = append(volumes, map[string]interface{}{
							"name": ackConfigValue.(map[string]interface{})["Name"],
							"path": ackConfigValue.(map[string]interface{})["Path"],
							"type": ackConfigValue.(map[string]interface{})["Type"],
						})
					}
					ackConfig["volumes"] = volumes
				}
				if value, exists := m["VolumeMounts"]; exists && len(value.([]interface{})) > 0 {
					var volumeMounts []map[string]interface{}
					for _, ackConfigValue := range value.([]interface{}) {
						volumeMounts = append(volumeMounts, map[string]interface{}{
							"name": ackConfigValue.(map[string]interface{})["Name"],
							"path": ackConfigValue.(map[string]interface{})["Path"],
						})
					}
					ackConfig["volume_mounts"] = volumeMounts
				}
				if value, exists := m["PreStartCommand"]; exists && len(value.([]interface{})) > 0 {
					var preStartCommands []string
					for _, ackConfigValue := range value.([]interface{}) {
						preStartCommands = append(preStartCommands, ackConfigValue.(string))
					}
					ackConfig["pre_start_command"] = preStartCommands
				}
				if value, exists := m["PodAffinity"]; exists && value.(string) != "" {
					ackConfig["pod_affinity"] = value
				}
				if value, exists := m["PodAntiAffinity"]; exists && value.(string) != "" {
					ackConfig["pod_anti_affinity"] = value
				}
				if value, exists := m["NodeAffinity"]; exists && value.(string) != "" {
					ackConfig["node_affinity"] = value
				}
				ackConfigs = append(ackConfigs, ackConfig)
				nodeGroup["ack_config"] = ackConfigs

				if ong, exists := oldNodeGroupsMap[nodeGroupMap["NodeGroupName"].(string)]; exists {
					if instValue, instExists := ong["instance_types"]; instExists {
						var instanceTypes []string
						for _, item := range instValue.(*schema.Set).List() {
							instanceTypes = append(instanceTypes, item.(string))
						}
						nodeGroup["instance_types"] = instanceTypes
					}
				}
			}

			if oldNodeGroup, exists := oldNodeGroupsMap[nodeGroupMap["NodeGroupName"].(string)]; exists {
				if v, ok := oldNodeGroup["deployment_set_strategy"]; ok && v.(string) != "" {
					deploymentSetStrategy := v.(string)
					if st, stExists := nodeGroupMap["DeploymentSetStrategy"]; stExists && st.(string) != "" {
						deploymentSetStrategy = st.(string)
					}
					nodeGroup["deployment_set_strategy"] = deploymentSetStrategy
				}
			}

			if oldNodeGroup, exists := oldNodeGroupsMap[nodeGroupMap["NodeGroupName"].(string)]; exists {
				if v, ok := oldNodeGroup["node_resize_strategy"]; ok && v.(string) != "" {
					nodeGroup["node_resize_strategy"] = v.(string)
				}

				if v, ok := oldNodeGroup["spot_strategy"]; ok && v.(string) != "" {
					nodeGroup["spot_strategy"] = v.(string)
				}

				if v, ok := oldNodeGroup["cost_optimized_config"]; ok && len(v.(*schema.Set).List()) > 0 {
					var costOptimizedConfig []map[string]interface{}
					for _, coc := range v.(*schema.Set).List() {
						costOptimizedConfig = append(costOptimizedConfig, coc.(map[string]interface{}))
					}
					nodeGroup["cost_optimized_config"] = costOptimizedConfig
				}
			}

			if v, ok := nodeGroupMap["SystemDisk"]; ok {
				systemDiskMap := v.(map[string]interface{})
				systemDisk := map[string]interface{}{
					"category": systemDiskMap["Category"],
					"size":     formatInt(systemDiskMap["Size"]),
				}
				if oldNodeGroup, exists := oldNodeGroupsMap[nodeGroupMap["NodeGroupName"].(string)]; exists {
					oldSystemDisk := oldNodeGroup["system_disk"].(*schema.Set).List()[0].(map[string]interface{})
					if v, exists := oldSystemDisk["performance_level"]; exists && v != nil && v.(string) != "" {
						systemDisk["performance_level"] = v
					}
					if v, exists := oldSystemDisk["count"]; exists && v != nil {
						if count := formatInt(v); count > 0 {
							systemDisk["count"] = count
						}
					}
				}
				nodeGroup["system_disk"] = []map[string]interface{}{systemDisk}
			}

			if v, ok := nodeGroupMap["DataDisks"]; ok && len(v.([]interface{})) > 0 {
				var dataDisks []map[string]interface{}
				for _, item := range v.([]interface{}) {
					dataDisksMap := item.(map[string]interface{})
					dataDisk := map[string]interface{}{
						"category": dataDisksMap["Category"],
						"size":     formatInt(dataDisksMap["Size"]),
						"count":    formatInt(dataDisksMap["Count"]),
					}
					if oldNodeGroup, exists := oldNodeGroupsMap[nodeGroupMap["NodeGroupName"].(string)]; exists {
						oldDataDisk := oldNodeGroup["data_disks"].(*schema.Set).List()[0].(map[string]interface{})
						if v, exists := oldDataDisk["performance_level"]; exists && v != nil && v.(string) != "" {
							dataDisk["performance_level"] = v
						}
						if v, exists := oldDataDisk["count"]; exists && v != nil && v.(int) != 0 {
							if count := formatInt(v); count > 0 {
								dataDisk["count"] = formatInt(v)
							}
						}
					}
					dataDisks = append(dataDisks, dataDisk)
				}
				nodeGroup["data_disks"] = dataDisks
			}

			nodeGroups = append(nodeGroups, nodeGroup)
		}
		sort.Slice(nodeGroups, func(i, j int) bool {
			return indexNodeGroupMap[nodeGroups[i]["node_group_name"].(string)] < indexNodeGroupMap[nodeGroups[j]["node_group_name"].(string)]
		})

		d.Set("node_groups", nodeGroups)
	}

	tags, err := emrService.ListTagResourcesNew(d.Id(), string(TagResourceCluster))

	if err != nil {
		return WrapError(err)
	}
	d.Set("tags", tagsToMap(tags))

	return nil
}