in ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java [167:522]
public Set<Resource> getResourcesAuthorized(Request request, Predicate predicate)
throws SystemException, UnsupportedPropertyException, NoSuchResourceException, NoSuchParentResourceException {
Set<Resource> resources = new HashSet<>();
final Set<ServiceComponentHostRequest> requests = new HashSet<>();
for (Map<String, Object> propertyMap : getPropertyMaps(predicate)) {
requests.add(getRequest(propertyMap));
}
Set<ServiceComponentHostResponse> responses = null;
try {
responses = getResources(new Command<Set<ServiceComponentHostResponse>>() {
@Override
public Set<ServiceComponentHostResponse> invoke() throws AmbariException {
return getManagementController().getHostComponents(requests);
}
});
} catch (Exception e) {
throw new SystemException("Failed to get components ", e);
}
Map<String,ServiceComponentHostResponse> componentMap = new HashMap<>();
// reduce set of sch responses to one sch response for every service component
for (ServiceComponentHostResponse resp: responses) {
String componentName = resp.getComponentName();
if (!componentMap.containsKey(componentName)) {
componentMap.put(resp.getComponentName(),resp);
}
}
ServiceComponentHostRequest schRequest = requests.iterator().next();
String requestComponentName = schRequest.getComponentName();
String requestServiceName = schRequest.getServiceName();
String requestHostName = schRequest.getHostname();
Map<String,List<ServiceComponentHostResponse>> serviceToComponentMap = new HashMap<>();
// sch response for the service components that have configFiles defined in the stack definition of the service
List <ServiceComponentHostResponse> schWithConfigFiles = new ArrayList<>();
Configuration configs = new Configuration();
Map<String, String> configMap = configs.getConfigsMap();
String TMP_PATH = configMap.get(Configuration.SERVER_TMP_DIR.getKey());
String pythonCmd = configMap.get(Configuration.AMBARI_PYTHON_WRAP.getKey());
List<String> pythonCompressFilesCmds = new ArrayList<>();
List<File> commandFiles = new ArrayList<>();
for (ServiceComponentHostResponse response : componentMap.values()){
AmbariManagementController managementController = getManagementController();
ConfigHelper configHelper = managementController.getConfigHelper();
Cluster cluster = null;
Clusters clusters = managementController.getClusters();
try {
cluster = clusters.getCluster(response.getClusterName());
String serviceName = response.getServiceName();
String componentName = response.getComponentName();
String hostName = response.getHostname();
String publicHostName = response.getPublicHostname();
ComponentInfo componentInfo = null;
String packageFolder = null;
Service service = cluster.getService(serviceName);
ServiceComponent component = service.getServiceComponent(componentName);
StackId stackId = component.getDesiredStackId();
componentInfo = managementController.getAmbariMetaInfo().
getComponent(stackId.getStackName(), stackId.getStackVersion(), serviceName, componentName);
packageFolder = managementController.getAmbariMetaInfo().
getService(stackId.getStackName(), stackId.getStackVersion(), serviceName).getServicePackageFolder();
String commandScript = componentInfo.getCommandScript().getScript();
List<ClientConfigFileDefinition> clientConfigFiles = componentInfo.getClientConfigFiles();
if (clientConfigFiles == null) {
if (componentMap.size() == 1) {
throw new SystemException("No configuration files defined for the component " + componentInfo.getName());
} else {
LOG.debug("No configuration files defined for the component {}", componentInfo.getName());
continue;
}
}
// service component hosts that have configFiles defined in the stack definition of the service
schWithConfigFiles.add(response);
if (serviceToComponentMap.containsKey(response.getServiceName())) {
List <ServiceComponentHostResponse> schResponseList = serviceToComponentMap.get(serviceName);
schResponseList.add(response);
} else {
List <ServiceComponentHostResponse> schResponseList = new ArrayList<>();
schResponseList.add(response);
serviceToComponentMap.put(serviceName,schResponseList);
}
String resourceDirPath = configs.getResourceDirPath();
String packageFolderAbsolute = resourceDirPath + File.separator + packageFolder;
String commandScriptAbsolute = packageFolderAbsolute + File.separator + commandScript;
Map<String, Map<String, String>> configurations = new TreeMap<>();
Map<String, Long> configVersions = new TreeMap<>();
Map<String, Map<PropertyType, Set<String>>> configPropertiesTypes = new TreeMap<>();
Map<String, Map<String, Map<String, String>>> configurationAttributes = new TreeMap<>();
Map<String, DesiredConfig> desiredClusterConfigs = cluster.getDesiredConfigs();
//Get configurations and configuration attributes
for (Map.Entry<String, DesiredConfig> desiredConfigEntry : desiredClusterConfigs.entrySet()) {
String configType = desiredConfigEntry.getKey();
DesiredConfig desiredConfig = desiredConfigEntry.getValue();
Config clusterConfig = cluster.getConfig(configType, desiredConfig.getTag());
if (clusterConfig != null) {
Map<String, String> props = new HashMap<>(clusterConfig.getProperties());
// Apply global properties for this host from all config groups
Map<String, Map<String, String>> allConfigTags = null;
allConfigTags = configHelper
.getEffectiveDesiredTags(cluster, schRequest.getHostname());
Map<String, Map<String, String>> configTags = new HashMap<>();
for (Map.Entry<String, Map<String, String>> entry : allConfigTags.entrySet()) {
if (entry.getKey().equals(clusterConfig.getType())) {
configTags.put(clusterConfig.getType(), entry.getValue());
}
}
Map<String, Map<String, String>> properties = configHelper
.getEffectiveConfigProperties(cluster, configTags);
if (!properties.isEmpty()) {
for (Map<String, String> propertyMap : properties.values()) {
props.putAll(propertyMap);
}
}
configurations.put(clusterConfig.getType(), props);
configVersions.put(clusterConfig.getType(), clusterConfig.getVersion());
configPropertiesTypes.put(clusterConfig.getType(), clusterConfig.getPropertiesTypes());
Map<String, Map<String, String>> attrs = new TreeMap<>();
configHelper.cloneAttributesMap(clusterConfig.getPropertiesAttributes(), attrs);
Map<String, Map<String, Map<String, String>>> attributes = configHelper
.getEffectiveConfigAttributes(cluster, configTags);
for (Map<String, Map<String, String>> attributesMap : attributes.values()) {
configHelper.cloneAttributesMap(attributesMap, attrs);
}
configurationAttributes.put(clusterConfig.getType(), attrs);
}
}
ConfigHelper.processHiddenAttribute(configurations, configurationAttributes, componentName, true);
for (Map.Entry<String, Map<String, Map<String, String>>> configurationAttributesEntry : configurationAttributes.entrySet()) {
Map<String, Map<String, String>> attrs = configurationAttributesEntry.getValue();
// remove internal attributes like "hidden"
attrs.remove("hidden");
}
// replace passwords on password references
for (Map.Entry<String, Map<String, String>> configEntry : configurations.entrySet()) {
String configType = configEntry.getKey();
Map<String, String> configProperties = configEntry.getValue();
Long configVersion = configVersions.get(configType);
Map<PropertyType, Set<String>> propertiesTypes = configPropertiesTypes.get(configType);
SecretReference.replacePasswordsWithReferences(propertiesTypes, configProperties, configType, configVersion);
}
Map<String, Set<String>> clusterHostInfo = null;
ServiceInfo serviceInfo = null;
String osFamily = null;
clusterHostInfo = StageUtils.getClusterHostInfo(cluster);
serviceInfo = managementController.getAmbariMetaInfo().getService(stackId.getStackName(),
stackId.getStackVersion(), serviceName);
try {
clusterHostInfo = StageUtils.substituteHostIndexes(clusterHostInfo);
} catch (AmbariException e) {
// Before moving substituteHostIndexes to StageUtils, a SystemException was thrown in the
// event an index could not be mapped to a host. After the move, this was changed to an
// AmbariException for consistency in the StageUtils class. To keep this method consistent
// with how it behaved in the past, if an AmbariException is thrown, it is caught and
// translated to a SystemException.
throw new SystemException(e.getMessage(), e);
}
osFamily = clusters.getHost(hostName).getOsFamily();
// Write down os specific info for the service
ServiceOsSpecific anyOs = null;
if (serviceInfo.getOsSpecifics().containsKey(AmbariMetaInfo.ANY_OS)) {
anyOs = serviceInfo.getOsSpecifics().get(AmbariMetaInfo.ANY_OS);
}
ServiceOsSpecific hostOs = populateServicePackagesInfo(serviceInfo, osFamily);
// Build package list that is relevant for host
List<ServiceOsSpecific.Package> packages =
new ArrayList<>();
if (anyOs != null) {
packages.addAll(anyOs.getPackages());
}
if (hostOs != null) {
packages.addAll(hostOs.getPackages());
}
String packageList = gson.toJson(packages);
String jsonConfigurations = null;
Map<String, Object> commandParams = new HashMap<>();
List<Map<String, String>> xmlConfigs = new LinkedList<>();
List<Map<String, String>> envConfigs = new LinkedList<>();
List<Map<String, String>> propertiesConfigs = new LinkedList<>();
//Fill file-dictionary configs from metainfo
for (ClientConfigFileDefinition clientConfigFile : clientConfigFiles) {
Map<String, String> fileDict = new HashMap<>();
fileDict.put(clientConfigFile.getFileName(), clientConfigFile.getDictionaryName());
if (clientConfigFile.getType().equals("xml")) {
xmlConfigs.add(fileDict);
} else if (clientConfigFile.getType().equals("env")) {
envConfigs.add(fileDict);
} else if (clientConfigFile.getType().equals("properties")) {
propertiesConfigs.add(fileDict);
}
}
TreeMap<String, String> clusterLevelParams = null;
TreeMap<String, String> ambariLevelParams = null;
TreeMap<String, String> topologyCommandParams = new TreeMap<>();
if (getManagementController() instanceof AmbariManagementControllerImpl){
AmbariManagementControllerImpl controller = ((AmbariManagementControllerImpl)getManagementController());
clusterLevelParams = controller.getMetadataClusterLevelParams(cluster, stackId);
ambariLevelParams = controller.getMetadataAmbariLevelParams();
Service s = cluster.getService(serviceName);
ServiceComponent sc = s.getServiceComponent(componentName);
ServiceComponentHost sch = sc.getServiceComponentHost(response.getHostname());
topologyCommandParams = controller.getTopologyCommandParams(cluster.getClusterId(), serviceName, componentName, sch);
}
TreeMap<String, String> agentLevelParams = new TreeMap<>();
agentLevelParams.put("hostname", hostName);
agentLevelParams.put("public_hostname", publicHostName);
commandParams.put(PACKAGE_LIST, packageList);
commandParams.put("xml_configs_list", xmlConfigs);
commandParams.put("env_configs_list", envConfigs);
commandParams.put("properties_configs_list", propertiesConfigs);
commandParams.put("output_file", componentName + "-configs" + Configuration.DEF_ARCHIVE_EXTENSION);
commandParams.putAll(topologyCommandParams);
Map<String, Object> jsonContent = new TreeMap<>();
jsonContent.put("configurations", configurations);
jsonContent.put("configurationAttributes", configurationAttributes);
jsonContent.put("commandParams", commandParams);
jsonContent.put("clusterHostInfo", clusterHostInfo);
jsonContent.put("ambariLevelParams", ambariLevelParams);
jsonContent.put("clusterLevelParams", clusterLevelParams);
jsonContent.put("agentLevelParams", agentLevelParams);
jsonContent.put("hostname", hostName);
jsonContent.put("public_hostname", publicHostName);
jsonContent.put("clusterName", cluster.getClusterName());
jsonContent.put("serviceName", serviceName);
jsonContent.put("role", componentName);
jsonContent.put("componentVersionMap", cluster.getComponentVersionMap());
jsonConfigurations = gson.toJson(jsonContent);
File tmpDirectory = new File(TMP_PATH);
if (!tmpDirectory.exists()) {
try {
tmpDirectory.mkdirs();
tmpDirectory.setWritable(true, true);
tmpDirectory.setReadable(true, true);
} catch (SecurityException se) {
throw new SystemException("Failed to get temporary directory to store configurations", se);
}
}
File jsonFile = File.createTempFile(componentName, "-configuration.json", tmpDirectory);
try {
jsonFile.setWritable(true, true);
jsonFile.setReadable(true, true);
} catch (SecurityException e) {
throw new SystemException("Failed to set permission", e);
}
PrintWriter printWriter = null;
try {
printWriter = new PrintWriter(jsonFile.getAbsolutePath());
printWriter.print(jsonConfigurations);
printWriter.close();
} catch (FileNotFoundException e) {
throw new SystemException("Failed to write configurations to json file ", e);
}
String cmd = pythonCmd + " " + commandScriptAbsolute + " generate_configs " + jsonFile.getAbsolutePath() + " " +
packageFolderAbsolute + " " + TMP_PATH + File.separator + "structured-out.json" + " INFO " + TMP_PATH;
commandFiles.add(jsonFile);
pythonCompressFilesCmds.add(cmd);
} catch (IOException e) {
throw new SystemException("Controller error ", e);
}
}
if (schWithConfigFiles.isEmpty()) {
throw new SystemException("No configuration files defined for any component" );
}
Integer totalCommands = pythonCompressFilesCmds.size() * 2;
Integer threadPoolSize = Math.min(totalCommands,configs.getExternalScriptThreadPoolSize());
ExecutorService processExecutor = Executors.newFixedThreadPool(threadPoolSize);
// put all threads that starts process to compress each component config files in the executor
try {
List<CommandLineThreadWrapper> pythonCmdThreads = executeCommands(processExecutor, pythonCompressFilesCmds);
// wait for all threads to finish
Integer timeout = configs.getExternalScriptTimeout();
waitForAllThreadsToJoin(processExecutor, pythonCmdThreads, timeout);
} finally {
for (File each : commandFiles) {
each.delete();
}
}
if (StringUtils.isEmpty(requestComponentName)) {
TarUtils tarUtils;
String fileName;
List <ServiceComponentHostResponse> schToTarConfigFiles = schWithConfigFiles;
if (StringUtils.isNotEmpty(requestHostName)) {
fileName = requestHostName + "(" + Resource.InternalType.Host.toString().toUpperCase()+")";
} else if (StringUtils.isNotEmpty(requestServiceName)) {
fileName = requestServiceName + "(" + Resource.InternalType.Service.toString().toUpperCase()+")";
schToTarConfigFiles = serviceToComponentMap.get(requestServiceName);
} else {
fileName = schRequest.getClusterName() + "(" + Resource.InternalType.Cluster.toString().toUpperCase()+")";
}
tarUtils = new TarUtils(TMP_PATH, fileName, schToTarConfigFiles);
tarUtils.tarConfigFiles();
}
Resource resource = new ResourceImpl(Resource.Type.ClientConfig);
resources.add(resource);
return resources;
}