in aliyun/log/logclient_operator.py [0:0]
def copy_project(from_client, to_client, from_project, to_project, copy_machine_group=False):
"""
copy project, logstore, machine group and logtail config to target project,
will create the target project if it doesn't exist
:type from_client: LogClient
:param from_client: logclient instance
:type to_client: LogClient
:param to_client: logclient instance
:type from_project: string
:param from_project: project name
:type to_project: string
:param to_project: project name
:type copy_machine_group: bool
:param copy_machine_group: if copy machine group resources, False by default.
:return:
"""
# copy project
ret = from_client.get_project(from_project)
try:
ret = to_client.create_project(to_project, ret.get_description())
except LogException as ex:
if ex.get_error_code() == 'ProjectAlreadyExist':
# don't create the project as it already exists
pass
else:
raise
default_fetch_size = 100
# list logstore and copy them
offset, size = 0, default_fetch_size
source_logstores, all_source_logstores = set(), set()
while True:
ret = from_client.list_logstore(from_project, offset=offset, size=size)
count = ret.get_logstores_count()
total = ret.get_logstores_total()
source_logstores = set(ret.get_logstores())
all_source_logstores = all_source_logstores.union(source_logstores)
for logstore_name in source_logstores:
# copy logstore
ret = from_client.get_logstore(from_project, logstore_name)
res_shard = from_client.list_shards(from_project, logstore_name)
expected_rwshard_count = len([shard for shard in res_shard.shards if shard['status'].lower() == 'readwrite'])
try:
ret2 = to_client.create_logstore(to_project, logstore_name, ret.get_ttl(),
min(expected_rwshard_count, MAX_INIT_SHARD_COUNT),
enable_tracking=ret.get_enable_tracking(),
append_meta=ret.append_meta,
auto_split=ret.auto_split,
max_split_shard=ret.max_split_shard,
preserve_storage=ret.preserve_storage
)
except LogException as ex:
if ex.get_error_code().lower() == "logstorealreadyexist":
pass
else:
raise
# copy index
index_config = None
try:
ret = from_client.get_index_config(from_project, logstore_name)
index_config = ret.get_index_config()
except LogException as ex:
if ex.get_error_code() == 'IndexConfigNotExist':
pass
else:
raise
if index_config is not None:
for x in range(60):
try:
ret2 = to_client.create_index(to_project, logstore_name, ret.get_index_config())
break
except LogException as ex:
if ex.get_error_code().lower() == "logstorenotexist" and x < 59:
time.sleep(1)
continue
if ex.get_error_code().lower() == "indexalreadyexist":
# target already has index, overwrite it
ret2 = to_client.update_index(to_project, logstore_name, index_config)
break
raise ex
offset += count
if count < size or offset >= total:
break
# list logtail config and copy them
offset, size = 0, default_fetch_size
source_configs = set()
while True:
ret = from_client.list_logtail_config(from_project, offset=offset, size=size)
count = ret.get_configs_count()
total = ret.get_configs_total()
for config_name in ret.get_configs():
ret = from_client.get_logtail_config(from_project, config_name)
if ret.logtail_config.logstore_name not in all_source_logstores:
continue
source_configs.add(config_name)
for x in range(60):
try:
ret2 = to_client.create_logtail_config(to_project, ret.logtail_config)
except LogException as ex:
if ex.get_error_code().lower() == "logstorenotexist" and x < 59:
time.sleep(1)
continue
if ex.get_error_code().lower() == "configalreadyexist":
break
raise ex
offset += count
if count < size or offset >= total:
break
# list machine group and copy them
offset, size = 0, default_fetch_size
while copy_machine_group:
ret = from_client.list_machine_group(from_project, offset=offset, size=size)
count = ret.get_machine_group_count()
total = ret.get_machine_group_total()
for group_name in ret.get_machine_group():
ret = from_client.get_machine_group(from_project, group_name)
try:
ret = to_client.create_machine_group(to_project, ret.get_machine_group())
except LogException as ex:
if ex.get_error_code() == 'MachineGroupAlreadyExist':
pass
else:
raise ex
# list all applied config and copy the relationship
ret = from_client.get_machine_group_applied_configs(from_project, group_name)
for config_name in ret.get_configs():
if config_name not in source_configs:
continue
for x in range(60):
try:
to_client.apply_config_to_machine_group(to_project, config_name, group_name)
except LogException as ex:
if ex.get_error_code().lower() in ("confignotexist", "groupnotexist") and x < 59:
time.sleep(1)
continue
raise ex
offset += count
if count < size or offset >= total:
break