in ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/scripts/yarn.py [0:0]
def yarn(name=None, config_dir=None):
"""
:param name: Component name, apptimelineserver, nodemanager, resourcemanager, or None (defaults for client)
:param config_dir: Which config directory to write configs to, which could be different during rolling upgrade.
"""
import params
install_lzo_if_needed()
if config_dir is None:
config_dir = params.hadoop_conf_dir
if params.yarn_nodemanager_recovery_dir:
Directory(
InlineTemplate(params.yarn_nodemanager_recovery_dir).get_content(),
owner=params.yarn_user,
group=params.user_group,
create_parents=True,
mode=0o755,
cd_access="a",
)
Directory(
[params.yarn_pid_dir_prefix, params.yarn_pid_dir, params.yarn_log_dir],
owner=params.yarn_user,
group=params.user_group,
create_parents=True,
cd_access="a",
)
Directory(
[
params.mapred_pid_dir_prefix,
params.mapred_pid_dir,
params.mapred_log_dir_prefix,
params.mapred_log_dir,
],
owner=params.mapred_user,
group=params.user_group,
create_parents=True,
cd_access="a",
)
Directory(
[params.yarn_log_dir_prefix],
owner=params.yarn_user,
group=params.user_group,
create_parents=True,
ignore_failures=True,
cd_access="a",
)
Directory(
params.yarn_hbase_conf_dir,
owner=params.yarn_hbase_user,
group=params.user_group,
create_parents=True,
cd_access="a",
)
Directory(
params.yarn_timeline_service_leveldb_state_store_path,
create_parents=True,
owner=params.yarn_user,
group=params.user_group,
)
# Some of these function calls depend on the directories above being created first.
if name == "resourcemanager":
setup_resourcemanager()
elif name == "nodemanager":
setup_nodemanager()
elif name == "apptimelineserver":
setup_ats()
elif name == "historyserver":
setup_historyserver()
elif name == "apptimelinereader":
if not params.use_external_hbase and not params.is_hbase_system_service_launch:
setup_atsv2_hbase_directories()
setup_atsv2_hbase_files()
XmlConfig(
"core-site.xml",
conf_dir=config_dir,
configurations=params.config["configurations"]["core-site"],
configuration_attributes=params.config["configurationAttributes"]["core-site"],
owner=params.hdfs_user,
group=params.user_group,
mode=0o644,
)
# During RU, Core Masters and Slaves need hdfs-site.xml
# TODO, instead of specifying individual configs, which is susceptible to breaking when new configs are added,
# RU should rely on all available in <stack-root>/<version>/hadoop/conf
XmlConfig(
"hdfs-site.xml",
conf_dir=config_dir,
configurations=params.config["configurations"]["hdfs-site"],
configuration_attributes=params.config["configurationAttributes"]["hdfs-site"],
owner=params.hdfs_user,
group=params.user_group,
mode=0o644,
)
XmlConfig(
"mapred-site.xml",
conf_dir=config_dir,
configurations=params.config["configurations"]["mapred-site"],
configuration_attributes=params.config["configurationAttributes"]["mapred-site"],
owner=params.yarn_user,
group=params.user_group,
mode=0o644,
)
configs = {}
configs.update(params.config["configurations"]["yarn-site"])
configs["hadoop.registry.dns.bind-port"] = params.config["configurations"][
"yarn-env"
]["registry.dns.bind-port"]
XmlConfig(
"yarn-site.xml",
conf_dir=config_dir,
configurations=configs,
configuration_attributes=params.config["configurationAttributes"]["yarn-site"],
owner=params.yarn_user,
group=params.user_group,
mode=0o644,
)
XmlConfig(
"capacity-scheduler.xml",
conf_dir=config_dir,
configurations=params.config["configurations"]["capacity-scheduler"],
configuration_attributes=params.config["configurationAttributes"][
"capacity-scheduler"
],
owner=params.yarn_user,
group=params.user_group,
mode=0o644,
)
if not isinstance(params.hbase_site_conf, UnknownConfiguration):
XmlConfig(
"hbase-site.xml",
conf_dir=params.yarn_hbase_conf_dir,
configurations=params.hbase_site_conf,
configuration_attributes=params.hbase_site_attributes,
owner=params.yarn_hbase_user,
group=params.user_group,
mode=0o644,
)
XmlConfig(
"resource-types.xml",
conf_dir=config_dir,
configurations=params.config["configurations"]["resource-types"],
configuration_attributes=params.config["configurationAttributes"]["resource-types"],
owner=params.yarn_user,
group=params.user_group,
mode=0o644,
)
File(
format("{limits_conf_dir}/yarn.conf"), mode=0o644, content=Template("yarn.conf.j2")
)
File(
format("{limits_conf_dir}/mapreduce.conf"),
mode=0o644,
content=Template("mapreduce.conf.j2"),
)
File(
os.path.join(config_dir, "yarn-env.sh"),
owner=params.yarn_user,
group=params.user_group,
mode=0o755,
content=InlineTemplate(params.yarn_env_sh_template),
)
File(
format("{yarn_container_bin}/container-executor"),
group=params.yarn_executor_container_group,
mode=params.container_executor_mode,
)
File(
os.path.join(config_dir, "container-executor.cfg"),
group=params.user_group,
mode=0o644,
content=InlineTemplate(params.container_executor_cfg_template),
)
Directory(
params.cgroups_dir,
group=params.user_group,
create_parents=True,
mode=0o755,
cd_access="a",
)
File(
os.path.join(config_dir, "mapred-env.sh"),
owner=params.tc_owner,
mode=0o755,
content=InlineTemplate(params.mapred_env_sh_template),
)
if params.security_enabled:
File(
os.path.join(params.hadoop_bin, "task-controller"),
owner="root",
group=params.mapred_tt_group,
mode=0o6050,
)
File(
os.path.join(config_dir, "taskcontroller.cfg"),
owner=params.tc_owner,
mode=params.tc_mode,
group=params.mapred_tt_group,
content=Template("taskcontroller.cfg.j2"),
)
File(
os.path.join(config_dir, "yarn_jaas.conf"),
owner=params.yarn_user,
group=params.user_group,
content=Template("yarn_jaas.conf.j2"),
)
if params.has_ats:
File(
os.path.join(config_dir, "yarn_ats_jaas.conf"),
owner=params.yarn_user,
group=params.user_group,
content=Template("yarn_ats_jaas.conf.j2"),
)
if params.has_registry_dns:
File(
os.path.join(config_dir, "yarn_registry_dns_jaas.conf"),
owner=params.yarn_user,
group=params.user_group,
mode=0o644,
content=Template("yarn_registry_dns_jaas.conf.j2"),
)
File(
os.path.join(config_dir, "yarn_nm_jaas.conf"),
owner=params.yarn_user,
group=params.user_group,
content=Template("yarn_nm_jaas.conf.j2"),
)
if params.has_hs:
File(
os.path.join(config_dir, "mapred_jaas.conf"),
owner=params.mapred_user,
group=params.user_group,
content=Template("mapred_jaas.conf.j2"),
)
else:
File(
os.path.join(config_dir, "taskcontroller.cfg"),
owner=params.tc_owner,
content=Template("taskcontroller.cfg.j2"),
)
XmlConfig(
"mapred-site.xml",
conf_dir=config_dir,
configurations=params.config["configurations"]["mapred-site"],
configuration_attributes=params.config["configurationAttributes"]["mapred-site"],
owner=params.mapred_user,
group=params.user_group,
)
XmlConfig(
"capacity-scheduler.xml",
conf_dir=config_dir,
configurations=params.config["configurations"]["capacity-scheduler"],
configuration_attributes=params.config["configurationAttributes"][
"capacity-scheduler"
],
owner=params.hdfs_user,
group=params.user_group,
)
if "ssl-client" in params.config["configurations"]:
XmlConfig(
"ssl-client.xml",
conf_dir=config_dir,
configurations=params.config["configurations"]["ssl-client"],
configuration_attributes=params.config["configurationAttributes"]["ssl-client"],
owner=params.hdfs_user,
group=params.user_group,
)
Directory(
params.hadoop_conf_secure_dir,
create_parents=True,
owner="root",
group=params.user_group,
cd_access="a",
)
XmlConfig(
"ssl-client.xml",
conf_dir=params.hadoop_conf_secure_dir,
configurations=params.config["configurations"]["ssl-client"],
configuration_attributes=params.config["configurationAttributes"]["ssl-client"],
owner=params.hdfs_user,
group=params.user_group,
)
if "ssl-server" in params.config["configurations"]:
XmlConfig(
"ssl-server.xml",
conf_dir=config_dir,
configurations=params.config["configurations"]["ssl-server"],
configuration_attributes=params.config["configurationAttributes"]["ssl-server"],
owner=params.hdfs_user,
group=params.user_group,
)
if os.path.exists(os.path.join(config_dir, "fair-scheduler.xml")):
File(
os.path.join(config_dir, "fair-scheduler.xml"),
owner=params.mapred_user,
group=params.user_group,
)
if os.path.exists(os.path.join(config_dir, "ssl-client.xml.example")):
File(
os.path.join(config_dir, "ssl-client.xml.example"),
owner=params.mapred_user,
group=params.user_group,
)
if os.path.exists(os.path.join(config_dir, "ssl-server.xml.example")):
File(
os.path.join(config_dir, "ssl-server.xml.example"),
owner=params.mapred_user,
group=params.user_group,
)
setup_atsv2_backend(name, config_dir)