in hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java [443:670]
public boolean init(String[] args) throws ParseException {
CommandLine cliParser = new GnuParser().parse(opts, args);
if (args.length == 0) {
throw new IllegalArgumentException("No args specified for client to initialize");
}
if (cliParser.hasOption("log_properties")) {
String log4jPath = cliParser.getOptionValue("log_properties");
try {
Log4jPropertyHelper.updateLog4jConfiguration(Client.class, log4jPath);
} catch (Exception e) {
LOG.warn("Can not set up custom log4j properties. " + e);
}
}
if (cliParser.hasOption("rolling_log_pattern")) {
rollingFilesPattern = cliParser.getOptionValue("rolling_log_pattern");
}
if (cliParser.hasOption("help")) {
printUsage();
return false;
}
if (cliParser.hasOption("debug")) {
debugFlag = true;
}
if (cliParser.hasOption("keep_containers_across_application_attempts")) {
LOG.info("keep_containers_across_application_attempts");
keepContainers = true;
}
if (cliParser.hasOption("placement_spec")) {
placementSpec = cliParser.getOptionValue("placement_spec");
// Check if it is parsable
PlacementSpec.parse(this.placementSpec);
}
appName = cliParser.getOptionValue("appname", "DistributedShell");
amPriority = Integer.parseInt(cliParser.getOptionValue("priority", "0"));
amQueue = cliParser.getOptionValue("queue", "default");
amMemory =
Integer.parseInt(cliParser.getOptionValue("master_memory", "-1"));
amVCores =
Integer.parseInt(cliParser.getOptionValue("master_vcores", "-1"));
if (cliParser.hasOption("master_resources")) {
Map<String, Long> masterResources =
parseResourcesString(cliParser.getOptionValue("master_resources"));
for (Map.Entry<String, Long> entry : masterResources.entrySet()) {
if (entry.getKey().equals(ResourceInformation.MEMORY_URI)) {
amMemory = entry.getValue();
} else if (entry.getKey().equals(ResourceInformation.VCORES_URI)) {
amVCores = entry.getValue().intValue();
} else {
amResources.put(entry.getKey(), entry.getValue());
}
}
}
amResourceProfile = cliParser.getOptionValue("master_resource_profile", "");
if (!cliParser.hasOption("jar")) {
throw new IllegalArgumentException("No jar file specified for application master");
}
appMasterJar = cliParser.getOptionValue("jar");
if (!cliParser.hasOption("shell_command") && !cliParser.hasOption("shell_script")) {
throw new IllegalArgumentException(
"No shell command or shell script specified to be executed by application master");
} else if (cliParser.hasOption("shell_command") && cliParser.hasOption("shell_script")) {
throw new IllegalArgumentException("Can not specify shell_command option " +
"and shell_script option at the same time");
} else if (cliParser.hasOption("shell_command")) {
shellCommand = cliParser.getOptionValue("shell_command");
} else {
shellScriptPath = cliParser.getOptionValue("shell_script");
}
if (cliParser.hasOption("shell_args")) {
shellArgs = cliParser.getOptionValues("shell_args");
}
if (cliParser.hasOption("shell_env")) {
String envs[] = cliParser.getOptionValues("shell_env");
for (String env : envs) {
env = env.trim();
int index = env.indexOf('=');
if (index == -1) {
shellEnv.put(env, "");
continue;
}
String key = env.substring(0, index);
String val = "";
if (index < (env.length()-1)) {
val = env.substring(index+1);
}
shellEnv.put(key, val);
}
}
shellCmdPriority = Integer.parseInt(cliParser.getOptionValue("shell_cmd_priority", "0"));
if (cliParser.hasOption("container_type")) {
String containerTypeStr = cliParser.getOptionValue("container_type");
if (Arrays.stream(ExecutionType.values()).noneMatch(
executionType -> executionType.toString()
.equals(containerTypeStr))) {
throw new IllegalArgumentException("Invalid container_type: "
+ containerTypeStr);
}
containerType = ExecutionType.valueOf(containerTypeStr);
}
if (cliParser.hasOption("promote_opportunistic_after_start")) {
autoPromoteContainers = true;
}
if (cliParser.hasOption("enforce_execution_type")) {
enforceExecType = true;
}
containerMemory =
Integer.parseInt(cliParser.getOptionValue("container_memory", "-1"));
containerVirtualCores =
Integer.parseInt(cliParser.getOptionValue("container_vcores", "-1"));
if (cliParser.hasOption("container_resources")) {
Map<String, Long> resources =
parseResourcesString(cliParser.getOptionValue("container_resources"));
for (Map.Entry<String, Long> entry : resources.entrySet()) {
if (entry.getKey().equals(ResourceInformation.MEMORY_URI)) {
containerMemory = entry.getValue();
} else if (entry.getKey().equals(ResourceInformation.VCORES_URI)) {
containerVirtualCores = entry.getValue().intValue();
} else {
containerResources.put(entry.getKey(), entry.getValue());
}
}
}
containerResourceProfile =
cliParser.getOptionValue("container_resource_profile", "");
numContainers =
Integer.parseInt(cliParser.getOptionValue("num_containers", "1"));
if (numContainers < 1) {
throw new IllegalArgumentException("Invalid no. of containers specified,"
+ " exiting. Specified numContainer=" + numContainers);
}
nodeLabelExpression = cliParser.getOptionValue("node_label_expression", () -> null);
clientTimeout = Integer.parseInt(cliParser.getOptionValue("timeout", "600000"));
attemptFailuresValidityInterval =
Long.parseLong(cliParser.getOptionValue(
"attempt_failures_validity_interval", "-1"));
log4jPropFile = cliParser.getOptionValue("log_properties", "");
// Get timeline domain options
if (cliParser.hasOption("domain")) {
domainId = cliParser.getOptionValue("domain");
toCreateDomain = cliParser.hasOption("create");
if (cliParser.hasOption("view_acls")) {
viewACLs = cliParser.getOptionValue("view_acls");
}
if (cliParser.hasOption("modify_acls")) {
modifyACLs = cliParser.getOptionValue("modify_acls");
}
}
// Get container retry options
if (cliParser.hasOption("container_retry_policy")) {
containerRetryOptions.add("--container_retry_policy "
+ cliParser.getOptionValue("container_retry_policy"));
}
if (cliParser.hasOption("container_retry_error_codes")) {
containerRetryOptions.add("--container_retry_error_codes "
+ cliParser.getOptionValue("container_retry_error_codes"));
}
if (cliParser.hasOption("container_max_retries")) {
containerRetryOptions.add("--container_max_retries "
+ cliParser.getOptionValue("container_max_retries"));
}
if (cliParser.hasOption("container_retry_interval")) {
containerRetryOptions.add("--container_retry_interval "
+ cliParser.getOptionValue("container_retry_interval"));
}
if (cliParser.hasOption("container_failures_validity_interval")) {
containerRetryOptions.add("--container_failures_validity_interval "
+ cliParser.getOptionValue("container_failures_validity_interval"));
}
if (cliParser.hasOption("flow_name")) {
flowName = cliParser.getOptionValue("flow_name");
}
if (cliParser.hasOption("flow_version")) {
flowVersion = cliParser.getOptionValue("flow_version");
}
if (cliParser.hasOption("flow_run_id")) {
try {
flowRunId = Long.parseLong(cliParser.getOptionValue("flow_run_id"));
} catch (NumberFormatException e) {
throw new IllegalArgumentException(
"Flow run is not a valid long value", e);
}
}
if (cliParser.hasOption("docker_client_config")) {
dockerClientConfig = cliParser.getOptionValue("docker_client_config");
}
if (cliParser.hasOption("application_tags")) {
String applicationTagsStr = cliParser.getOptionValue("application_tags");
String[] appTags = applicationTagsStr.split(",");
for (String appTag : appTags) {
this.applicationTags.add(appTag.trim());
}
}
if (cliParser.hasOption("localize_files")) {
String filesStr = cliParser.getOptionValue("localize_files");
if (filesStr.contains(",")) {
String[] files = filesStr.split(",");
filesToLocalize = Arrays.asList(files);
} else {
filesToLocalize.add(filesStr);
}
}
return true;
}