in contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/stack_advisor.py [0:0]
def recommendRangerConfigurations(self, configurations, clusterData, services, hosts):
servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
putRangerEnvProperty = self.putProperty(configurations, "ranger-env", services)
putRangerAdminProperty = self.putProperty(configurations, "admin-properties", services)
putRangerAdminSiteProperty = self.putProperty(configurations, "ranger-admin-site", services)
putRangerUgsyncSite = self.putProperty(configurations, "ranger-ugsync-site", services)
putTagsyncAppProperty = self.putProperty(configurations, "tagsync-application-properties", services)
putTagsyncSiteProperty = self.putProperty(configurations, "ranger-tagsync-site", services)
# get zookeeper hosts
zookeeper_host_port = self.getZKHostPortString(services)
# Build policymgr_external_url
protocol = 'http'
ranger_admin_host = 'localhost'
port = '6080'
# Check if http is disabled. For HDF this can be checked in ranger-admin-site/ranger.service.http.enabled
if ('ranger-admin-site' in services['configurations'] and 'ranger.service.http.enabled' in services['configurations']['ranger-admin-site']['properties'] \
and services['configurations']['ranger-admin-site']['properties']['ranger.service.http.enabled'].lower() == 'false'):
# HTTPS protocol is used
protocol = 'https'
if 'ranger-admin-site' in services['configurations'] and \
'ranger.service.https.port' in services['configurations']['ranger-admin-site']['properties']:
port = services['configurations']['ranger-admin-site']['properties']['ranger.service.https.port']
else:
# HTTP protocol is used
if 'ranger-admin-site' in services['configurations'] and \
'ranger.service.http.port' in services['configurations']['ranger-admin-site']['properties']:
port = services['configurations']['ranger-admin-site']['properties']['ranger.service.http.port']
ranger_admin_hosts = self.getComponentHostNames(services, "RANGER", "RANGER_ADMIN")
if ranger_admin_hosts:
if len(ranger_admin_hosts) > 1 \
and services['configurations'] \
and 'admin-properties' in services['configurations'] and 'policymgr_external_url' in services['configurations']['admin-properties']['properties'] \
and services['configurations']['admin-properties']['properties']['policymgr_external_url'] \
and services['configurations']['admin-properties']['properties']['policymgr_external_url'].strip():
# in case of HA deployment keep the policymgr_external_url specified in the config
policymgr_external_url = services['configurations']['admin-properties']['properties']['policymgr_external_url']
else:
ranger_admin_host = ranger_admin_hosts[0]
policymgr_external_url = f"{protocol}://{ranger_admin_host}:{port}"
putRangerAdminProperty('policymgr_external_url', policymgr_external_url)
cluster_env = getServicesSiteProperties(services, "cluster-env")
security_enabled = cluster_env is not None and "security_enabled" in cluster_env and \
cluster_env["security_enabled"].lower() == "true"
if "ranger-env" in configurations and not security_enabled:
putRangerEnvProperty("ranger-storm-plugin-enabled", "No")
if 'admin-properties' in services['configurations'] and ('DB_FLAVOR' in services['configurations']['admin-properties']['properties']) \
and ('db_host' in services['configurations']['admin-properties']['properties']) and ('db_name' in services['configurations']['admin-properties']['properties']):
rangerDbFlavor = services['configurations']["admin-properties"]["properties"]["DB_FLAVOR"]
rangerDbHost = services['configurations']["admin-properties"]["properties"]["db_host"]
rangerDbName = services['configurations']["admin-properties"]["properties"]["db_name"]
ranger_db_url_dict = {
'MYSQL': {'ranger.jpa.jdbc.driver': 'com.mysql.jdbc.Driver',
'ranger.jpa.jdbc.url': 'jdbc:mysql://' + self.getDBConnectionHostPort(rangerDbFlavor, rangerDbHost) + '/' + rangerDbName},
'ORACLE': {'ranger.jpa.jdbc.driver': 'oracle.jdbc.driver.OracleDriver',
'ranger.jpa.jdbc.url': 'jdbc:oracle:thin:@//' + self.getOracleDBConnectionHostPort(rangerDbFlavor, rangerDbHost, rangerDbName)},
'POSTGRES': {'ranger.jpa.jdbc.driver': 'org.postgresql.Driver',
'ranger.jpa.jdbc.url': 'jdbc:postgresql://' + self.getDBConnectionHostPort(rangerDbFlavor, rangerDbHost) + '/' + rangerDbName},
'MSSQL': {'ranger.jpa.jdbc.driver': 'com.microsoft.sqlserver.jdbc.SQLServerDriver',
'ranger.jpa.jdbc.url': 'jdbc:sqlserver://' + self.getDBConnectionHostPort(rangerDbFlavor, rangerDbHost) + ';databaseName=' + rangerDbName},
'SQLA': {'ranger.jpa.jdbc.driver': 'sap.jdbc4.sqlanywhere.IDriver',
'ranger.jpa.jdbc.url': 'jdbc:sqlanywhere:host=' + self.getDBConnectionHostPort(rangerDbFlavor, rangerDbHost) + ';database=' + rangerDbName}
}
rangerDbProperties = ranger_db_url_dict.get(rangerDbFlavor, ranger_db_url_dict['MYSQL'])
for key in rangerDbProperties:
putRangerAdminSiteProperty(key, rangerDbProperties.get(key))
if 'admin-properties' in services['configurations'] and ('DB_FLAVOR' in services['configurations']['admin-properties']['properties']) \
and ('db_host' in services['configurations']['admin-properties']['properties']):
rangerDbFlavor = services['configurations']["admin-properties"]["properties"]["DB_FLAVOR"]
rangerDbHost = services['configurations']["admin-properties"]["properties"]["db_host"]
ranger_db_privelege_url_dict = {
'MYSQL': {'ranger_privelege_user_jdbc_url': 'jdbc:mysql://' + self.getDBConnectionHostPort(rangerDbFlavor, rangerDbHost)},
'ORACLE': {'ranger_privelege_user_jdbc_url': 'jdbc:oracle:thin:@//' + self.getOracleDBConnectionHostPort(rangerDbFlavor, rangerDbHost, None)},
'POSTGRES': {'ranger_privelege_user_jdbc_url': 'jdbc:postgresql://' + self.getDBConnectionHostPort(rangerDbFlavor, rangerDbHost) + '/postgres'},
'MSSQL': {'ranger_privelege_user_jdbc_url': 'jdbc:sqlserver://' + self.getDBConnectionHostPort(rangerDbFlavor, rangerDbHost) + ';'},
'SQLA': {'ranger_privelege_user_jdbc_url': 'jdbc:sqlanywhere:host=' + self.getDBConnectionHostPort(rangerDbFlavor, rangerDbHost) + ';'}
}
rangerPrivelegeDbProperties = ranger_db_privelege_url_dict.get(rangerDbFlavor, ranger_db_privelege_url_dict['MYSQL'])
for key in rangerPrivelegeDbProperties:
putRangerEnvProperty(key, rangerPrivelegeDbProperties.get(key))
# Recommend ldap settings based on ambari.properties configuration
if 'ambari-server-properties' in services and \
'ambari.ldap.isConfigured' in services['ambari-server-properties'] and \
services['ambari-server-properties']['ambari.ldap.isConfigured'].lower() == "true":
serverProperties = services['ambari-server-properties']
if 'authentication.ldap.baseDn' in serverProperties:
putRangerUgsyncSite('ranger.usersync.ldap.searchBase', serverProperties['authentication.ldap.baseDn'])
if 'authentication.ldap.groupMembershipAttr' in serverProperties:
putRangerUgsyncSite('ranger.usersync.group.memberattributename', serverProperties['authentication.ldap.groupMembershipAttr'])
if 'authentication.ldap.groupNamingAttr' in serverProperties:
putRangerUgsyncSite('ranger.usersync.group.nameattribute', serverProperties['authentication.ldap.groupNamingAttr'])
if 'authentication.ldap.groupObjectClass' in serverProperties:
putRangerUgsyncSite('ranger.usersync.group.objectclass', serverProperties['authentication.ldap.groupObjectClass'])
if 'authentication.ldap.managerDn' in serverProperties:
putRangerUgsyncSite('ranger.usersync.ldap.binddn', serverProperties['authentication.ldap.managerDn'])
if 'authentication.ldap.primaryUrl' in serverProperties:
ldap_protocol = 'ldap://'
if 'authentication.ldap.useSSL' in serverProperties and serverProperties['authentication.ldap.useSSL'] == 'true':
ldap_protocol = 'ldaps://'
ldapUrl = ldap_protocol + serverProperties['authentication.ldap.primaryUrl'] if serverProperties['authentication.ldap.primaryUrl'] else serverProperties['authentication.ldap.primaryUrl']
putRangerUgsyncSite('ranger.usersync.ldap.url', ldapUrl)
if 'authentication.ldap.userObjectClass' in serverProperties:
putRangerUgsyncSite('ranger.usersync.ldap.user.objectclass', serverProperties['authentication.ldap.userObjectClass'])
if 'authentication.ldap.usernameAttribute' in serverProperties:
putRangerUgsyncSite('ranger.usersync.ldap.user.nameattribute', serverProperties['authentication.ldap.usernameAttribute'])
# Recommend Ranger Authentication method
authMap = {
'org.apache.ranger.unixusersync.process.UnixUserGroupBuilder': 'UNIX',
'org.apache.ranger.ldapusersync.process.LdapUserGroupBuilder': 'LDAP'
}
if 'ranger-ugsync-site' in services['configurations'] and 'ranger.usersync.source.impl.class' in services['configurations']["ranger-ugsync-site"]["properties"]:
rangerUserSyncClass = services['configurations']["ranger-ugsync-site"]["properties"]["ranger.usersync.source.impl.class"]
if rangerUserSyncClass in authMap:
rangerAuthMethod = authMap.get(rangerUserSyncClass)
putRangerAdminSiteProperty('ranger.authentication.method', rangerAuthMethod)
# Recommend Ambari Infra Solr properties
is_solr_cloud_enabled = False
if 'ranger-env' in services['configurations'] and 'is_solrCloud_enabled' in services['configurations']["ranger-env"]["properties"]:
is_solr_cloud_enabled = services['configurations']["ranger-env"]["properties"]["is_solrCloud_enabled"] == "true"
is_external_solr_cloud_enabled = False
if 'ranger-env' in services['configurations'] and 'is_external_solrCloud_enabled' in services['configurations']['ranger-env']['properties']:
is_external_solr_cloud_enabled = services['configurations']['ranger-env']['properties']['is_external_solrCloud_enabled'] == 'true'
ranger_audit_zk_port = ''
if 'AMBARI_INFRA' in servicesList and zookeeper_host_port and is_solr_cloud_enabled and not is_external_solr_cloud_enabled:
zookeeper_host_port = zookeeper_host_port.split(',')
zookeeper_host_port.sort()
zookeeper_host_port = ",".join(zookeeper_host_port)
infra_solr_znode = '/infra-solr'
if 'infra-solr-env' in services['configurations'] and \
('infra_solr_znode' in services['configurations']['infra-solr-env']['properties']):
infra_solr_znode = services['configurations']['infra-solr-env']['properties']['infra_solr_znode']
ranger_audit_zk_port = f'{zookeeper_host_port}{infra_solr_znode}'
putRangerAdminSiteProperty('ranger.audit.solr.zookeepers', ranger_audit_zk_port)
elif zookeeper_host_port and is_solr_cloud_enabled and is_external_solr_cloud_enabled:
ranger_audit_zk_port = f'{zookeeper_host_port}/ranger_audits'
putRangerAdminSiteProperty('ranger.audit.solr.zookeepers', ranger_audit_zk_port)
else:
putRangerAdminSiteProperty('ranger.audit.solr.zookeepers', 'NONE')
# Recommend Ranger supported service's audit properties
ranger_services = [
{'service_name': 'KAFKA', 'audit_file': 'ranger-kafka-audit'},
{'service_name': 'STORM', 'audit_file': 'ranger-storm-audit'},
{'service_name': 'NIFI', 'audit_file': 'ranger-nifi-audit'}
]
for item in range(len(ranger_services)):
if ranger_services[item]['service_name'] in servicesList:
component_audit_file = ranger_services[item]['audit_file']
if component_audit_file in services["configurations"]:
ranger_audit_dict = [
{'filename': 'ranger-env', 'configname': 'xasecure.audit.destination.solr', 'target_configname': 'xasecure.audit.destination.solr'},
{'filename': 'ranger-admin-site', 'configname': 'ranger.audit.solr.urls', 'target_configname': 'xasecure.audit.destination.solr.urls'},
{'filename': 'ranger-admin-site', 'configname': 'ranger.audit.solr.zookeepers', 'target_configname': 'xasecure.audit.destination.solr.zookeepers'}
]
putRangerAuditProperty = self.putProperty(configurations, component_audit_file, services)
for item in ranger_audit_dict:
if item['filename'] in services["configurations"] and item['configname'] in services["configurations"][item['filename']]["properties"]:
if item['filename'] in configurations and item['configname'] in configurations[item['filename']]["properties"]:
rangerAuditProperty = configurations[item['filename']]["properties"][item['configname']]
else:
rangerAuditProperty = services["configurations"][item['filename']]["properties"][item['configname']]
putRangerAuditProperty(item['target_configname'], rangerAuditProperty)
ranger_plugins_serviceuser = [
{'service_name': 'STORM', 'file_name': 'storm-env', 'config_name': 'storm_user', 'target_configname': 'ranger.plugins.storm.serviceuser'},
{'service_name': 'KAFKA', 'file_name': 'kafka-env', 'config_name': 'kafka_user', 'target_configname': 'ranger.plugins.kafka.serviceuser'},
{'service_name': 'NIFI', 'file_name': 'nifi-env', 'config_name': 'nifi_user', 'target_configname': 'ranger.plugins.nifi.serviceuser'}
]
for item in range(len(ranger_plugins_serviceuser)):
if ranger_plugins_serviceuser[item]['service_name'] in servicesList:
file_name = ranger_plugins_serviceuser[item]['file_name']
config_name = ranger_plugins_serviceuser[item]['config_name']
target_configname = ranger_plugins_serviceuser[item]['target_configname']
if file_name in services["configurations"] and config_name in services["configurations"][file_name]["properties"]:
service_user = services["configurations"][file_name]["properties"][config_name]
putRangerAdminSiteProperty(target_configname, service_user)
has_ranger_tagsync = False
if 'RANGER' in servicesList:
ranger_tagsync_host = self.getComponentHostNames(services, "RANGER", "RANGER_TAGSYNC")
has_ranger_tagsync = len(ranger_tagsync_host) > 0
if 'ATLAS' in servicesList and has_ranger_tagsync:
putTagsyncSiteProperty('ranger.tagsync.source.atlas', 'true')
if zookeeper_host_port and has_ranger_tagsync:
putTagsyncAppProperty('atlas.kafka.zookeeper.connect', zookeeper_host_port)
if 'KAFKA' in servicesList and has_ranger_tagsync:
kafka_hosts = self.getHostNamesWithComponent("KAFKA", "KAFKA_BROKER", services)
kafka_port = '6667'
if 'kafka-broker' in services['configurations'] and (
'port' in services['configurations']['kafka-broker']['properties']):
kafka_port = services['configurations']['kafka-broker']['properties']['port']
kafka_host_port = []
for i in range(len(kafka_hosts)):
kafka_host_port.append(kafka_hosts[i] + ':' + kafka_port)
final_kafka_host = ",".join(kafka_host_port)
putTagsyncAppProperty('atlas.kafka.bootstrap.servers', final_kafka_host)