bigtop-deploy/puppet/modules/hadoop/templates/hdfs-site.xml

<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

<!-- Licensed to the Apache Software Foundation (ASF) under one or more       -->
<!-- contributor license agreements.  See the NOTICE file distributed with    -->
<!-- this work for additional information regarding copyright ownership.      -->
<!-- The ASF licenses this file to You under the Apache License, Version 2.0  -->
<!-- (the "License"); you may not use this file except in compliance with     -->
<!-- the License.  You may obtain a copy of the License at                    -->
<!--                                                                          -->
<!--     http://www.apache.org/licenses/LICENSE-2.0                           -->
<!--                                                                          -->
<!-- Unless required by applicable law or agreed to in writing, software      -->
<!-- distributed under the License is distributed on an "AS IS" BASIS,        -->
<!-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -->
<!-- See the License for the specific language governing permissions and      -->
<!-- limitations under the License.                                           -->

<% namenode_hosts = Array(@hadoop_namenode_host) -%>
<configuration>

<% if @ha != "disabled" -%>
  <!-- HA -->

<%   if @ha == "auto" -%>
  <property>
    <name>dfs.ha.automatic-failover.enabled</name>
    <value>true</value>
  </property>
<%   end -%>

  <property> 
    <name>dfs.nameservices</name>
    <value><%= @nameservice_id %></value>
  </property>

  <property>
    <name>dfs.ha.namenodes.<%= @nameservice_id %></name>
    <value><%= (1..namenode_hosts.length).map { |n| "nn#{n}" }.join(",") %></value>
  </property>

<%   namenode_hosts.each_with_index do |host,idx| -%>
  <property>
    <name>dfs.namenode.rpc-address.<%= @nameservice_id %>.nn<%= idx+1 %></name>
    <value><%= host %>:<%= @hadoop_namenode_port %></value>
  </property>
 
<%     if not @hadoop_namenode_bind_host.nil? -%>
  <property>
    <name>dfs.namenode.rpc-bind-host.<%= @nameservice_id %>.nn<%= idx+1 %></name>
    <value><%= @hadoop_namenode_bind_host %></value>
  </property>

<%     end -%>
  <property>
    <name>dfs.namenode.http-address.<%= @nameservice_id %>.nn<%= idx+1 %></name>
    <value><%= host %>:<%= @hadoop_namenode_http_port %></value>
  </property>

<%     if not @hadoop_namenode_http_bind_host.nil? -%>
  <property>
    <name>dfs.namenode.http-bind-host.<%= @nameservice_id %>.nn<%= idx+1 %></name>
    <value><%= @hadoop_namenode_http_bind_host %></value>
  </property>

<%     end -%>
  <property>
    <name>dfs.namenode.https-address.<%= @nameservice_id %>.nn<%= idx+1 %></name>
    <value><%= host %>:<%= @hadoop_namenode_https_port %></value>
  </property>

<%     if not @hadoop_namenode_https_bind_host.nil? -%>
  <property>
    <name>dfs.namenode.https-bind-host.<%= @nameservice_id %>.nn<%= idx+1 %></name>
    <value><%= @hadoop_namenode_https_bind_host %></value>
  </property>

<%     end -%>
<%   end -%>
<%   if @shared_edits_dir -%>
  <property>
    <name>dfs.namenode.shared.edits.dir</name>
    <value><%= @shared_edits_dir %></value>
  </property>

<%   end -%>
  <property>
    <name>dfs.client.failover.proxy.provider.<%= @nameservice_id %></name>
    <value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
  </property>

<%   if @sshfence_privkey -%>
  <property>
    <name>dfs.ha.fencing.methods</name>
    <value>sshfence(<%= @sshfence_user %>)</value>
  </property>

  <property>
    <name>dfs.ha.fencing.ssh.private-key-files</name>
    <value><%= @sshfence_keypath %></value>
  </property>

<%   else -%>
  <property>
    <name>dfs.ha.fencing.methods</name>
    <value></value>
  </property>

<%   end -%>
<% else -%>
  <!-- non HA -->

  <property>
    <name>dfs.namenode.rpc-address</name>
    <value><%= namenode_hosts[0] %>:<%= @hadoop_namenode_port %></value>
  </property>

<%   if not @hadoop_namenode_bind_host.nil? -%>
  <property>
    <name>dfs.namenode.rpc-bind-host</name>
    <value><%= @hadoop_namenode_bind_host %></value>
  </property>

<%   end -%>
  <property>
    <name>dfs.namenode.http-address</name>
    <value><%= namenode_hosts[0] %>:<%= @hadoop_namenode_http_port %></value>
  </property>

<%   if not @hadoop_namenode_http_bind_host.nil? -%>
  <property>
    <name>dfs.namenode.http-bind-host</name>
    <value><%= @hadoop_namenode_http_bind_host %></value>
  </property>

<%   end -%>
  <property>
    <name>dfs.namenode.https-address</name>
    <value><%= namenode_hosts[0] %>:<%= @hadoop_namenode_https_port %></value>
  </property>

<%   if not @hadoop_namenode_https_bind_host.nil? -%>
  <property>
    <name>dfs.namenode.https-bind-host</name>
    <value><%= @hadoop_namenode_https_bind_host %></value>
  </property>

<%   end -%>
<%   if @hadoop_security_authentication == "kerberos" -%>
  <property>
    <name>dfs.block.access.token.enable</name>
    <value>true</value>
  </property>
  <property>
    <name>dfs.http.policy</name>
    <value>HTTPS_ONLY</value>
  </property>
  
  <!-- NameNode security config -->
  <property>
    <name>dfs.https.address</name>
    <value><%= namenode_hosts[0] %>:50475</value>
  </property>
  <property>
    <name>dfs.https.port</name>
    <value>50475</value>
  </property>
  <property>
    <name>dfs.namenode.keytab.file</name>
    <value>/etc/hdfs.keytab</value> <!-- path to the HDFS keytab -->
  </property>
  <property>
    <name>dfs.namenode.kerberos.principal</name>
    <value>hdfs/_HOST@<%= @kerberos_realm %></value>
  </property>
  <property>
    <name>dfs.namenode.kerberos.https.principal</name>
    <value>host/_HOST@<%= @kerberos_realm %></value>
  </property>
  <property>
    <name>dfs.web.authentication.kerberos.keytab</name>
    <value>/etc/hdfs.keytab</value> <!-- path to the HDFS keytab -->
  </property>
  <property>
    <name>dfs.web.authentication.kerberos.principal</name>
    <value>HTTP/_HOST@<%= @kerberos_realm %></value>
  </property>

  
  <!-- Secondary NameNode security config -->
  <property>
    <name>dfs.secondary.http.address</name>
    <value><%= namenode_hosts[0] %>:0</value>
  </property>
  <property>
    <name>dfs.secondary.https.address</name>
    <value><%= namenode_hosts[0] %>:50495</value>
  </property>
  <property>
    <name>dfs.secondary.https.port</name>
    <value>50495</value>
  </property>
  <property>
    <name>dfs.secondary.namenode.keytab.file</name>
    <value>/etc/hdfs.keytab</value> <!-- path to the HDFS keytab -->
  </property>
  <property>
    <name>dfs.secondary.namenode.kerberos.principal</name>
    <value>hdfs/_HOST@<%= @kerberos_realm %></value>
  </property>
  <property>
    <name>dfs.secondary.namenode.kerberos.https.principal</name>
    <value>host/_HOST@<%= @kerberos_realm %></value>
  </property>
  
  <!-- DataNode security config -->
  <property>
    <name>dfs.datanode.data.dir.perm</name>
    <value>700</value> 
  </property>
  <property>
    <name>dfs.data.transfer.protection</name>
    <value>privacy</value>
  </property>
  <property>
    <name>dfs.datanode.keytab.file</name>
    <value>/etc/hdfs.keytab</value> <!-- path to the HDFS keytab -->
  </property>
  <property>
    <name>dfs.datanode.kerberos.principal</name>
    <value>hdfs/_HOST@<%= @kerberos_realm %></value>
  </property>
  <property>
    <name>dfs.datanode.kerberos.https.principal</name>
    <value>host/_HOST@<%= @kerberos_realm %></value>
  </property>

<%   end -%>
<% end -%>
  <property>
    <name>dfs.datanode.data.dir</name>
    <value><%= @hdfs_data_dirs.map { |dir| "file://#{dir}" }.join(",") %></value>
  </property>

<% if @hdfs_replace_datanode_on_failure %>
  <property>
    <name>dfs.client.block.write.replace-datanode-on-failure.enable</name>
    <value><%= @hdfs_replace_datanode_on_failure %></value>
  </property>
<% end %>

<% if @hdfs_support_append %>
  <property>
    <name>dfs.support.append</name>
    <value><%= @hdfs_support_append %></value>
  </property>
<% end %>

<% if @hdfs_shortcut_reader %>
  <property>
    <name>dfs.client.read.shortcircuit</name>
    <value>true</value>
  </property>

  <property>
    <name>dfs.domain.socket.path</name>
    <value>/var/lib/hadoop-hdfs/dn_socket</value>
  </property>
<% end %>
 
  <property>
    <name>dfs.namenode.name.dir</name>
    <value><%= @namenode_data_dirs.map { |dir| "file://#{dir}" }.join(",") %></value>
  </property>

  <property>
    <name>dfs.permissions.superusergroup</name>
    <value>hadoop</value>
    <description>The name of the group of super-users.</description>
  </property>

  <!-- increase the number of datanode transceivers way above the default of 256
     - this is for hbase -->
  <property>
    <name>dfs.datanode.max.xcievers</name>
    <value>4096</value>
  </property>

  <!-- Configurations for large cluster -->
<% if @hadoop_config_dfs_block_size -%>
  <property>
    <name>dfs.block.size</name>
    <value><%= @hadoop_config_dfs_block_size %></value>
  </property>

<% end -%>
<% if @hadoop_config_namenode_handler_count -%>
  <property>
    <name>dfs.namenode.handler.count</name>
    <value><%= @hadoop_config_namenode_handler_count %></value>
  </property>

<% end -%>
  <property>
    <name>dfs.webhdfs.enabled</name>
    <value><%= @hdfs_webhdfs_enabled %></value>
  </property>

<% if @hdfs_datanode_fsdataset_volume_choosing_policy -%>
  <property>
    <name>dfs.datanode.fsdataset.volume.choosing.policy</name>
    <value><%= @hdfs_datanode_fsdataset_volume_choosing_policy %></value>
  </property>

<% end -%>
<% if @hdfs_replication -%>
  <property>
    <name>dfs.replication</name>
    <value><%= @hdfs_replication %></value>
  </property>

<% end -%>
<% if @shared_edits_dir.start_with?("qjournal://") -%>
<%   if @journalnode_edits_dir -%>
  <property>
    <name>dfs.journalnode.edits.dir</name>
    <value><%= @journalnode_edits_dir %></value>
  </property>

<%   end -%>
<%   if @journalnode_host -%>
  <property>
    <name>dfs.journalnode.rpc-address</name>
    <value><%= @journalnode_host %>:<%= @journalnode_port %></value>
  </property>

  <property>
    <name>dfs.journalnode.http-address</name>
    <value><%= @journalnode_host %>:<%= @journalnode_http_port %></value>
  </property>

  <property>
    <name>dfs.journalnode.https-address</name>
    <value><%= @journalnode_host %>:<%= @journalnode_https_port %></value>
  </property>

<%   end -%>
<% end -%>

<% if not @namenode_datanode_registration_ip_hostname_check.nil? -%>
  <property>
    <name>dfs.namenode.datanode.registration.ip-hostname-check</name>
    <value><%= @namenode_datanode_registration_ip_hostname_check %></value>
  </property>
<% end -%>

</configuration>