bool FileDiscoveryOptions::Init()

in core/file_server/FileDiscoveryOptions.cpp [135:345]


bool FileDiscoveryOptions::Init(const Json::Value& config,
                                const CollectionPipelineContext& ctx,
                                const string& pluginType) {
    string errorMsg;

    // FilePaths + MaxDirSearchDepth
    if (!GetMandatoryListParam<string>(config, "FilePaths", mFilePaths, errorMsg)) {
        PARAM_ERROR_RETURN(ctx.GetLogger(),
                           ctx.GetAlarm(),
                           errorMsg,
                           pluginType,
                           ctx.GetConfigName(),
                           ctx.GetProjectName(),
                           ctx.GetLogstoreName(),
                           ctx.GetRegion());
    }
    if (mFilePaths.size() != 1) {
        PARAM_ERROR_RETURN(ctx.GetLogger(),
                           ctx.GetAlarm(),
                           "list param FilePaths has more than 1 element",
                           pluginType,
                           ctx.GetConfigName(),
                           ctx.GetProjectName(),
                           ctx.GetLogstoreName(),
                           ctx.GetRegion());
    }
    auto dirAndFile = GetDirAndFileNameFromPath(mFilePaths[0]);
    mBasePath = dirAndFile.first, mFilePattern = dirAndFile.second;
    if (mBasePath.empty() || mFilePattern.empty()) {
        PARAM_ERROR_RETURN(ctx.GetLogger(),
                           ctx.GetAlarm(),
                           "string param FilePaths[0] is invalid",
                           pluginType,
                           ctx.GetConfigName(),
                           ctx.GetProjectName(),
                           ctx.GetLogstoreName(),
                           ctx.GetRegion());
    }
#if defined(_MSC_VER)
    mBasePath = EncodingConverter::GetInstance()->FromUTF8ToACP(mBasePath);
    mFilePattern = EncodingConverter::GetInstance()->FromUTF8ToACP(mFilePattern);
#endif
    size_t len = mBasePath.size();
    if (len > 2 && mBasePath[len - 1] == '*' && mBasePath[len - 2] == '*'
        && mBasePath[len - 3] == filesystem::path::preferred_separator) {
        if (len == 3) {
            // for parent path like /**, we should maintain root dir, i.e., /
            mBasePath = mBasePath.substr(0, len - 2);
        } else {
            mBasePath = mBasePath.substr(0, len - 3);
        }
        // MaxDirSearchDepth is only valid when parent path ends with **
        if (!GetOptionalIntParam(config, "MaxDirSearchDepth", mMaxDirSearchDepth, errorMsg)) {
            PARAM_WARNING_DEFAULT(ctx.GetLogger(),
                                  ctx.GetAlarm(),
                                  errorMsg,
                                  mMaxDirSearchDepth,
                                  pluginType,
                                  ctx.GetConfigName(),
                                  ctx.GetProjectName(),
                                  ctx.GetLogstoreName(),
                                  ctx.GetRegion());
        }
    }
    ParseWildcardPath();

    // PreservedDirDepth
    if (!GetOptionalIntParam(config, "PreservedDirDepth", mPreservedDirDepth, errorMsg)) {
        PARAM_WARNING_DEFAULT(ctx.GetLogger(),
                              ctx.GetAlarm(),
                              errorMsg,
                              mPreservedDirDepth,
                              pluginType,
                              ctx.GetConfigName(),
                              ctx.GetProjectName(),
                              ctx.GetLogstoreName(),
                              ctx.GetRegion());
    }

    // ExcludeFilePaths
    if (!GetOptionalListParam<string>(config, "ExcludeFilePaths", mExcludeFilePaths, errorMsg)) {
        PARAM_WARNING_IGNORE(ctx.GetLogger(),
                             ctx.GetAlarm(),
                             errorMsg,
                             pluginType,
                             ctx.GetConfigName(),
                             ctx.GetProjectName(),
                             ctx.GetLogstoreName(),
                             ctx.GetRegion());
    } else {
        for (size_t i = 0; i < mExcludeFilePaths.size(); ++i) {
            if (!filesystem::path(mExcludeFilePaths[i]).is_absolute()) {
                PARAM_WARNING_IGNORE(ctx.GetLogger(),
                                     ctx.GetAlarm(),
                                     "string param ExcludeFilePaths[" + ToString(i) + "] is not absolute",
                                     pluginType,
                                     ctx.GetConfigName(),
                                     ctx.GetProjectName(),
                                     ctx.GetLogstoreName(),
                                     ctx.GetRegion());
                continue;
            }
            bool isMultipleLevelWildcard = mExcludeFilePaths[i].find("**") != string::npos;
            if (isMultipleLevelWildcard) {
                mMLFilePathBlacklist.push_back(mExcludeFilePaths[i]);
            } else {
                mFilePathBlacklist.push_back(mExcludeFilePaths[i]);
            }
        }
    }

    // ExcludeFiles
    if (!GetOptionalListParam<string>(config, "ExcludeFiles", mExcludeFiles, errorMsg)) {
        PARAM_WARNING_IGNORE(ctx.GetLogger(),
                             ctx.GetAlarm(),
                             errorMsg,
                             pluginType,
                             ctx.GetConfigName(),
                             ctx.GetProjectName(),
                             ctx.GetLogstoreName(),
                             ctx.GetRegion());
    } else {
        for (size_t i = 0; i < mExcludeFiles.size(); ++i) {
            if (mExcludeFiles[i].find(filesystem::path::preferred_separator) != string::npos) {
                PARAM_WARNING_IGNORE(ctx.GetLogger(),
                                     ctx.GetAlarm(),
                                     "string param ExcludeFiles[" + ToString(i) + "] contains path separator",
                                     pluginType,
                                     ctx.GetConfigName(),
                                     ctx.GetProjectName(),
                                     ctx.GetLogstoreName(),
                                     ctx.GetRegion());
                continue;
            }
            mFileNameBlacklist.push_back(mExcludeFiles[i]);
        }
    }

    // ExcludeDirs
    if (!GetOptionalListParam<string>(config, "ExcludeDirs", mExcludeDirs, errorMsg)) {
        PARAM_WARNING_IGNORE(ctx.GetLogger(),
                             ctx.GetAlarm(),
                             errorMsg,
                             pluginType,
                             ctx.GetConfigName(),
                             ctx.GetProjectName(),
                             ctx.GetLogstoreName(),
                             ctx.GetRegion());
    } else {
        for (size_t i = 0; i < mExcludeDirs.size(); ++i) {
            if (!filesystem::path(mExcludeDirs[i]).is_absolute()) {
                PARAM_WARNING_IGNORE(ctx.GetLogger(),
                                     ctx.GetAlarm(),
                                     "string param ExcludeDirs[" + ToString(i) + "] is not absolute",
                                     pluginType,
                                     ctx.GetConfigName(),
                                     ctx.GetProjectName(),
                                     ctx.GetLogstoreName(),
                                     ctx.GetRegion());
                continue;
            }
            bool isMultipleLevelWildcard = mExcludeDirs[i].find("**") != string::npos;
            if (isMultipleLevelWildcard) {
                mMLWildcardDirPathBlacklist.push_back(mExcludeDirs[i]);
                continue;
            }
            bool isWildcardPath
                = mExcludeDirs[i].find("*") != string::npos || mExcludeDirs[i].find("?") != string::npos;
            if (isWildcardPath) {
                mWildcardDirPathBlacklist.push_back(mExcludeDirs[i]);
            } else {
                mDirPathBlacklist.push_back(mExcludeDirs[i]);
            }
        }
    }
    if (!mDirPathBlacklist.empty() || !mWildcardDirPathBlacklist.empty() || !mMLWildcardDirPathBlacklist.empty()
        || !mMLFilePathBlacklist.empty() || !mFileNameBlacklist.empty() || !mFilePathBlacklist.empty()) {
        mHasBlacklist = true;
    }

    // AllowingCollectingFilesInRootDir
    if (!GetOptionalBoolParam(
            config, "AllowingCollectingFilesInRootDir", mAllowingCollectingFilesInRootDir, errorMsg)) {
        PARAM_WARNING_DEFAULT(ctx.GetLogger(),
                              ctx.GetAlarm(),
                              errorMsg,
                              mAllowingCollectingFilesInRootDir,
                              pluginType,
                              ctx.GetConfigName(),
                              ctx.GetProjectName(),
                              ctx.GetLogstoreName(),
                              ctx.GetRegion());
    } else if (mAllowingCollectingFilesInRootDir) {
        BOOL_FLAG(enable_root_path_collection) = mAllowingCollectingFilesInRootDir;
    }

    // AllowingIncludedByMultiConfigs
    if (!GetOptionalBoolParam(config, "AllowingIncludedByMultiConfigs", mAllowingIncludedByMultiConfigs, errorMsg)) {
        PARAM_WARNING_DEFAULT(ctx.GetLogger(),
                              ctx.GetAlarm(),
                              errorMsg,
                              mAllowingIncludedByMultiConfigs,
                              pluginType,
                              ctx.GetConfigName(),
                              ctx.GetProjectName(),
                              ctx.GetLogstoreName(),
                              ctx.GetRegion());
    }

    return true;
}