in ambari-infra-solr-client/src/main/python/migrationConfigGenerator.py [0:0]
def generate_ambari_solr_migration_ini_file(options, accessor, protocol):
print("Start generating config file: {0} ...".format(options.ini_file))
config = configparser.RawConfigParser()
config.add_section('ambari_server')
config.set('ambari_server', 'host', options.host)
config.set('ambari_server', 'port', options.port)
config.set('ambari_server', 'cluster', options.cluster)
config.set('ambari_server', 'protocol', protocol)
config.set('ambari_server', 'username', options.username)
config.set('ambari_server', 'password', options.password)
print("Get Ambari cluster details ...")
blueprint = get_json(accessor, CLUSTERS_URL.format(options.cluster) + BLUEPRINT_CONFIG_URL)
installed_components = get_installed_components(blueprint)
print("Set JAVA_HOME: {0}".format(options.java_home))
host = socket.getfqdn()
cluster_config = get_cluster_configs(blueprint)
solr_hosts = get_solr_hosts(options, accessor)
if solr_hosts and host not in solr_hosts:
print("{0}WARNING{1}: Host '{2}' is not found in Infra Solr hosts ({3}). Migration commands won't work from here."\
.format(colors.WARNING, colors.ENDC, host, ','.join(solr_hosts)))
zookeeper_hosts = get_zookeeper_server_hosts(options, accessor)
security_enabled = is_security_enabled(cluster_config)
zk_connect_string = get_zookeeper_connection_string(cluster_config, zookeeper_hosts)
if zk_connect_string:
print("Service detected: " + colors.OKGREEN + "ZOOKEEPER" + colors.ENDC)
print("Zookeeper connection string: {0}".format(str(zk_connect_string)))
solr_protocol = get_solr_protocol(cluster_config)
solr_urls = get_solr_urls(cluster_config, solr_hosts, solr_protocol)
if solr_urls:
print("Service detected: " + colors.OKGREEN + "AMBARI_INFRA_SOLR" + colors.ENDC)
solr_znode = get_solr_znode(cluster_config)
if solr_znode:
print("Infra Solr znode: {0}".format(solr_znode))
infra_solr_env_props = get_config_props(cluster_config, 'infra-solr-env')
infra_solr_user = infra_solr_env_props['infra_solr_user'] if 'infra_solr_user' in infra_solr_env_props else 'infra-solr'
infra_solr_kerberos_keytab = infra_solr_env_props['infra_solr_kerberos_keytab'] if 'infra_solr_kerberos_keytab' in infra_solr_env_props else '/etc/security/keytabs/ambari-infra-solr.service.keytab'
infra_solr_kerberos_principal_config = infra_solr_env_props['infra_solr_kerberos_principal'] if 'infra_solr_kerberos_principal' in infra_solr_env_props else 'infra-solr'
infra_solr_kerberos_principal = "infra-solr/" + host
if '/' in infra_solr_kerberos_principal_config:
infra_solr_kerberos_principal = infra_solr_kerberos_principal_config.replace('_HOST',host)
else:
infra_solr_kerberos_principal = infra_solr_kerberos_principal_config + "/" + host
infra_solr_port = infra_solr_env_props['infra_solr_port'] if 'infra_solr_port' in infra_solr_env_props else '8886'
config.add_section('local')
config.set('local', 'java_home', options.java_home)
config.set('local', 'hostname', host)
if options.shared_drive:
config.set('local', 'shared_drive', 'true')
else:
config.set('local', 'shared_drive', 'false')
config.add_section('cluster')
config.set('cluster', 'kerberos_enabled', security_enabled)
config.add_section('infra_solr')
config.set('infra_solr', 'protocol', solr_protocol)
config.set('infra_solr', 'hosts', ','.join(solr_hosts))
config.set('infra_solr', 'zk_connect_string', zk_connect_string)
config.set('infra_solr', 'znode', solr_znode)
config.set('infra_solr', 'user', infra_solr_user)
config.set('infra_solr', 'port', infra_solr_port)
if security_enabled == 'true':
config.set('infra_solr', 'keytab', infra_solr_kerberos_keytab)
config.set('infra_solr', 'principal', infra_solr_kerberos_principal)
zookeeper_env_props = get_config_props(cluster_config, 'zookeeper-env')
zookeeper_principal_name = zookeeper_env_props['zookeeper_principal_name'] if 'zookeeper_principal_name' in zookeeper_env_props else "zookeeper/_HOST@EXAMPLE.COM"
zk_principal_user = zookeeper_principal_name.split("/")[0]
default_zk_quorum = "{zookeeper_quorum}"
external_zk_connection_string = infra_solr_env_props['infra_solr_zookeeper_quorum'] if 'infra_solr_zookeeper_quorum' in infra_solr_env_props else default_zk_quorum
if default_zk_quorum != external_zk_connection_string:
print("Found external zk connection string: {0}".format(external_zk_connection_string))
config.set('infra_solr', 'external_zk_connect_string', external_zk_connection_string)
config.set('infra_solr', 'zk_principal_user', zk_principal_user)
state_json_map = retry(get_state_json_map, solr_urls, infra_solr_user, security_enabled, infra_solr_kerberos_keytab, infra_solr_kerberos_principal, count=options.retry, delay=options.delay, context="Get clusterstate.json")
coll_shard_map=get_shard_numbers_per_collections(state_json_map)
max_shards_map=get_max_shards_for_collections(state_json_map)
config.add_section('ranger_collection')
if "RANGER_ADMIN" in installed_components and not options.skip_ranger:
print("Service detected: " + colors.OKGREEN + "RANGER" + colors.ENDC)
ranger_env_props = get_config_props(cluster_config, 'ranger-env')
if "is_solrCloud_enabled" in ranger_env_props and ranger_env_props['is_solrCloud_enabled'] == 'true':
if "is_external_solrCloud_enabled" in ranger_env_props and ranger_env_props['is_external_solrCloud_enabled'] == 'true' and not options.force_ranger:
config.set('ranger_collection', 'enabled', 'false')
else:
config.set('ranger_collection', 'enabled', 'true')
ranger_config_set = ranger_env_props['ranger_solr_config_set'] if ranger_env_props and 'ranger_solr_config_set' in ranger_env_props else 'ranger_audits'
ranger_collection_name = ranger_env_props['ranger_solr_collection_name'] if ranger_env_props and 'ranger_solr_collection_name' in ranger_env_props else 'ranger_audits'
config.set('ranger_collection', 'ranger_config_set_name', ranger_config_set)
config.set('ranger_collection', 'ranger_collection_name', ranger_collection_name)
if ranger_collection_name in coll_shard_map:
config.set('ranger_collection', 'ranger_collection_shards', coll_shard_map[ranger_collection_name])
if ranger_collection_name in max_shards_map:
config.set('ranger_collection', 'ranger_collection_max_shards_per_node', max_shards_map[ranger_collection_name])
config.set('ranger_collection', 'backup_ranger_config_set_name', 'old_ranger_audits')
config.set('ranger_collection', 'backup_ranger_collection_name', 'old_ranger_audits')
print('Ranger Solr collection: ' + ranger_collection_name)
ranger_backup_path = None
if options.backup_base_path:
ranger_backup_path = os.path.join(options.backup_base_path, "ranger")
elif options.backup_ranger_base_path:
ranger_backup_path = options.backup_ranger_base_path
if ranger_backup_path is not None:
config.set('ranger_collection', 'backup_path', ranger_backup_path)
print('Ranger backup path: ' + ranger_backup_path)
if options.ranger_hdfs_base_path:
config.set('ranger_collection', 'hdfs_base_path', options.ranger_hdfs_base_path)
elif options.hdfs_base_path:
config.set('ranger_collection', 'hdfs_base_path', options.hdfs_base_path)
else:
config.set('ranger_collection', 'enabled', 'false')
else:
config.set('ranger_collection', 'enabled', 'false')
config.add_section('atlas_collections')
if "ATLAS_SERVER" in installed_components and not options.skip_atlas:
print("Service detected: " + colors.OKGREEN + "ATLAS" + colors.ENDC)
config.set('atlas_collections', 'enabled', 'true')
config.set('atlas_collections', 'config_set', 'atlas_configs')
config.set('atlas_collections', 'fulltext_index_name', 'fulltext_index')
config.set('atlas_collections', 'backup_fulltext_index_name', 'old_fulltext_index')
if 'fulltext_index' in coll_shard_map:
config.set('atlas_collections', 'fulltext_index_shards', coll_shard_map['fulltext_index'])
if 'fulltext_index' in max_shards_map:
config.set('atlas_collections', 'fulltext_index_max_shards_per_node', max_shards_map['fulltext_index'])
config.set('atlas_collections', 'edge_index_name', 'edge_index')
config.set('atlas_collections', 'backup_edge_index_name', 'old_edge_index')
if 'edge_index' in coll_shard_map:
config.set('atlas_collections', 'edge_index_shards', coll_shard_map['edge_index'])
if 'edge_index' in max_shards_map:
config.set('atlas_collections', 'edge_index_max_shards_per_node', max_shards_map['edge_index'])
config.set('atlas_collections', 'vertex_index_name', 'vertex_index')
config.set('atlas_collections', 'backup_vertex_index_name', 'old_vertex_index')
if 'vertex_index' in coll_shard_map:
config.set('atlas_collections', 'vertex_index_shards', coll_shard_map['vertex_index'])
if 'vertex_index' in max_shards_map:
config.set('atlas_collections', 'vertex_index_max_shards_per_node', max_shards_map['vertex_index'])
print('Atlas Solr collections: fulltext_index, edge_index, vertex_index')
atlas_backup_path = None
if options.backup_base_path:
atlas_backup_path = os.path.join(options.backup_base_path, "atlas")
elif options.backup_ranger_base_path:
atlas_backup_path = options.backup_atlas_base_path
if atlas_backup_path is not None:
config.set('atlas_collections', 'backup_path', atlas_backup_path)
print('Atlas backup path: ' + atlas_backup_path)
if options.atlas_hdfs_base_path:
config.set('atlas_collections', 'hdfs_base_path', options.atlas_hdfs_base_path)
elif options.hdfs_base_path:
config.set('atlas_collections', 'hdfs_base_path', options.hdfs_base_path)
else:
config.set('atlas_collections', 'enabled', 'false')
config.add_section('logsearch_collections')
if "LOGSEARCH_SERVER" in installed_components:
print("Service detected: " + colors.OKGREEN + "LOGSEARCH" + colors.ENDC)
logsearch_props = get_config_props(cluster_config, 'logsearch-properties')
logsearch_hadoop_logs_coll_name = logsearch_props['logsearch.solr.collection.service.logs'] if logsearch_props and 'logsearch.solr.collection.service.logs' in logsearch_props else 'hadoop_logs'
logsearch_audit_logs_coll_name = logsearch_props['logsearch.solr.collection.audit.logs'] if logsearch_props and 'logsearch.solr.collection.audit.logs' in logsearch_props else 'audit_logs'
config.set('logsearch_collections', 'enabled', 'true')
config.set('logsearch_collections', 'hadoop_logs_collection_name', logsearch_hadoop_logs_coll_name)
config.set('logsearch_collections', 'audit_logs_collection_name', logsearch_audit_logs_coll_name)
config.set('logsearch_collections', 'history_collection_name', 'history')
print('Log Search Solr collections: {0}, {1}, history'.format(logsearch_hadoop_logs_coll_name, logsearch_audit_logs_coll_name))
else:
config.set('logsearch_collections', 'enabled', 'false')
if security_enabled == 'true':
print("Kerberos: enabled")
else:
print("Kerberos: disabled")
with open(options.ini_file, 'w') as f:
config.write(f)
print("Config file generation has finished " + colors.OKGREEN + "successfully" + colors.ENDC)