def copy_logstore()

in aliyun/log/logclient_operator.py [0:0]

• 74 lines of code
• 19 McCabe index (conditional complexity)

def copy_logstore(from_client, from_project, from_logstore, to_logstore, to_project=None, to_client=None, to_region_endpoint=None, keep_config_name=False):
    """
    copy logstore, index, logtail config to target logstore, machine group are not included yet.
    the target logstore will be crated if not existing

    :type from_client: LogClient
    :param from_client: logclient instance

    :type from_project: string
    :param from_project: project name

    :type from_logstore: string
    :param from_logstore: logstore name

    :type to_logstore: string
    :param to_logstore: target logstore name

    :type to_project: string
    :param to_project: project name, copy to same project if not being specified, will try to create it if not being specified

    :type to_client: LogClient
    :param to_client: logclient instance, use it to operate on the "to_project" if being specified

    :type to_region_endpoint: string
    :param to_region_endpoint: target region, use it to operate on the "to_project" while "to_client" not be specified
    
    :type keep_config_name: bool
    :param keep_config_name: use the same logtail config name as the source one, note that the config name must be unique in the same project

    :return:
    """

    if to_region_endpoint is not None and to_client is None:
        to_client = copy.deepcopy(from_client)
        to_client.set_endpoint(to_region_endpoint)
    else:
        to_client = to_client or from_client

    # check client
    if to_project is not None:
        # check if target project exists or not
        ret = from_client.get_project(from_project)
        try:
            ret = to_client.create_project(to_project, ret.get_description())
        except LogException as ex:
            if ex.get_error_code() == 'ProjectAlreadyExist':
                # don't create the project as it already exists
                pass
            else:
                raise

    to_project = to_project or from_project

    # return if logstore are the same one
    if from_client is to_client and from_project == to_project and from_logstore == to_logstore:
        return

    # copy logstore
    ret = from_client.get_logstore(from_project, from_logstore)
    res_shard = from_client.list_shards(from_project, from_logstore)
    expected_rwshard_count = len([shard for shard in res_shard.shards if shard['status'].lower() == 'readwrite'])
    try:
        ret = to_client.create_logstore(to_project, to_logstore,
                                        ttl=ret.get_ttl(),
                                        shard_count=min(expected_rwshard_count, MAX_INIT_SHARD_COUNT),
                                        enable_tracking=ret.get_enable_tracking(),
                                        append_meta=ret.append_meta,
                                        auto_split=ret.auto_split,
                                        max_split_shard=ret.max_split_shard,
                                        preserve_storage=ret.preserve_storage)
    except LogException as ex:
        if ex.get_error_code().lower() == "logstorealreadyexist":
            # update logstore's settings
            ret = to_client.update_logstore(to_project, to_logstore,
                                            ttl=ret.get_ttl(),
                                            enable_tracking=ret.get_enable_tracking(),
                                            append_meta=ret.append_meta,
                                            auto_split=ret.auto_split,
                                            max_split_shard=ret.max_split_shard,
                                            preserve_storage=ret.preserve_storage
                                            )

            # arrange shard to expected count
            res = arrange_shard(to_client, to_project, to_logstore, min(expected_rwshard_count, MAX_INIT_SHARD_COUNT))
        else:
            raise


    # copy index
    try:
        ret = from_client.get_index_config(from_project, from_logstore)
        ret = to_client.create_index(to_project, to_logstore, ret.get_index_config())
    except LogException as ex:
        if ex.get_error_code() == 'IndexConfigNotExist':
            # source has no index
            pass
        elif ex.get_error_code() == 'IndexAlreadyExist':
            # target already has index, overwrite it
            ret = to_client.update_index(to_project, to_logstore, ret.get_index_config())
            pass
        else:
            raise

    # list logtail config linked to the logstore and copy them
    default_fetch_size = 100
    offset, size = 0, default_fetch_size
    while True:
        ret = from_client.list_logtail_config(from_project, offset=offset, size=size)
        count = ret.get_configs_count()
        total = ret.get_configs_total()

        for config_name in ret.get_configs():
            ret = from_client.get_logtail_config(from_project, config_name)
            config = ret.logtail_config
            if config.logstore_name != from_logstore:
                continue
            if keep_config_name:
                config.config_name = config_name
            else:
                config.config_name = to_logstore + '_' + config_name
            config.logstore_name = to_logstore
            ret = to_client.create_logtail_config(to_project, config)

        offset += count
        if count < size or offset >= total:
            break