in core/file_server/FileDiscoveryOptions.cpp [135:345]
bool FileDiscoveryOptions::Init(const Json::Value& config,
const CollectionPipelineContext& ctx,
const string& pluginType) {
string errorMsg;
// FilePaths + MaxDirSearchDepth
if (!GetMandatoryListParam<string>(config, "FilePaths", mFilePaths, errorMsg)) {
PARAM_ERROR_RETURN(ctx.GetLogger(),
ctx.GetAlarm(),
errorMsg,
pluginType,
ctx.GetConfigName(),
ctx.GetProjectName(),
ctx.GetLogstoreName(),
ctx.GetRegion());
}
if (mFilePaths.size() != 1) {
PARAM_ERROR_RETURN(ctx.GetLogger(),
ctx.GetAlarm(),
"list param FilePaths has more than 1 element",
pluginType,
ctx.GetConfigName(),
ctx.GetProjectName(),
ctx.GetLogstoreName(),
ctx.GetRegion());
}
auto dirAndFile = GetDirAndFileNameFromPath(mFilePaths[0]);
mBasePath = dirAndFile.first, mFilePattern = dirAndFile.second;
if (mBasePath.empty() || mFilePattern.empty()) {
PARAM_ERROR_RETURN(ctx.GetLogger(),
ctx.GetAlarm(),
"string param FilePaths[0] is invalid",
pluginType,
ctx.GetConfigName(),
ctx.GetProjectName(),
ctx.GetLogstoreName(),
ctx.GetRegion());
}
#if defined(_MSC_VER)
mBasePath = EncodingConverter::GetInstance()->FromUTF8ToACP(mBasePath);
mFilePattern = EncodingConverter::GetInstance()->FromUTF8ToACP(mFilePattern);
#endif
size_t len = mBasePath.size();
if (len > 2 && mBasePath[len - 1] == '*' && mBasePath[len - 2] == '*'
&& mBasePath[len - 3] == filesystem::path::preferred_separator) {
if (len == 3) {
// for parent path like /**, we should maintain root dir, i.e., /
mBasePath = mBasePath.substr(0, len - 2);
} else {
mBasePath = mBasePath.substr(0, len - 3);
}
// MaxDirSearchDepth is only valid when parent path ends with **
if (!GetOptionalIntParam(config, "MaxDirSearchDepth", mMaxDirSearchDepth, errorMsg)) {
PARAM_WARNING_DEFAULT(ctx.GetLogger(),
ctx.GetAlarm(),
errorMsg,
mMaxDirSearchDepth,
pluginType,
ctx.GetConfigName(),
ctx.GetProjectName(),
ctx.GetLogstoreName(),
ctx.GetRegion());
}
}
ParseWildcardPath();
// PreservedDirDepth
if (!GetOptionalIntParam(config, "PreservedDirDepth", mPreservedDirDepth, errorMsg)) {
PARAM_WARNING_DEFAULT(ctx.GetLogger(),
ctx.GetAlarm(),
errorMsg,
mPreservedDirDepth,
pluginType,
ctx.GetConfigName(),
ctx.GetProjectName(),
ctx.GetLogstoreName(),
ctx.GetRegion());
}
// ExcludeFilePaths
if (!GetOptionalListParam<string>(config, "ExcludeFilePaths", mExcludeFilePaths, errorMsg)) {
PARAM_WARNING_IGNORE(ctx.GetLogger(),
ctx.GetAlarm(),
errorMsg,
pluginType,
ctx.GetConfigName(),
ctx.GetProjectName(),
ctx.GetLogstoreName(),
ctx.GetRegion());
} else {
for (size_t i = 0; i < mExcludeFilePaths.size(); ++i) {
if (!filesystem::path(mExcludeFilePaths[i]).is_absolute()) {
PARAM_WARNING_IGNORE(ctx.GetLogger(),
ctx.GetAlarm(),
"string param ExcludeFilePaths[" + ToString(i) + "] is not absolute",
pluginType,
ctx.GetConfigName(),
ctx.GetProjectName(),
ctx.GetLogstoreName(),
ctx.GetRegion());
continue;
}
bool isMultipleLevelWildcard = mExcludeFilePaths[i].find("**") != string::npos;
if (isMultipleLevelWildcard) {
mMLFilePathBlacklist.push_back(mExcludeFilePaths[i]);
} else {
mFilePathBlacklist.push_back(mExcludeFilePaths[i]);
}
}
}
// ExcludeFiles
if (!GetOptionalListParam<string>(config, "ExcludeFiles", mExcludeFiles, errorMsg)) {
PARAM_WARNING_IGNORE(ctx.GetLogger(),
ctx.GetAlarm(),
errorMsg,
pluginType,
ctx.GetConfigName(),
ctx.GetProjectName(),
ctx.GetLogstoreName(),
ctx.GetRegion());
} else {
for (size_t i = 0; i < mExcludeFiles.size(); ++i) {
if (mExcludeFiles[i].find(filesystem::path::preferred_separator) != string::npos) {
PARAM_WARNING_IGNORE(ctx.GetLogger(),
ctx.GetAlarm(),
"string param ExcludeFiles[" + ToString(i) + "] contains path separator",
pluginType,
ctx.GetConfigName(),
ctx.GetProjectName(),
ctx.GetLogstoreName(),
ctx.GetRegion());
continue;
}
mFileNameBlacklist.push_back(mExcludeFiles[i]);
}
}
// ExcludeDirs
if (!GetOptionalListParam<string>(config, "ExcludeDirs", mExcludeDirs, errorMsg)) {
PARAM_WARNING_IGNORE(ctx.GetLogger(),
ctx.GetAlarm(),
errorMsg,
pluginType,
ctx.GetConfigName(),
ctx.GetProjectName(),
ctx.GetLogstoreName(),
ctx.GetRegion());
} else {
for (size_t i = 0; i < mExcludeDirs.size(); ++i) {
if (!filesystem::path(mExcludeDirs[i]).is_absolute()) {
PARAM_WARNING_IGNORE(ctx.GetLogger(),
ctx.GetAlarm(),
"string param ExcludeDirs[" + ToString(i) + "] is not absolute",
pluginType,
ctx.GetConfigName(),
ctx.GetProjectName(),
ctx.GetLogstoreName(),
ctx.GetRegion());
continue;
}
bool isMultipleLevelWildcard = mExcludeDirs[i].find("**") != string::npos;
if (isMultipleLevelWildcard) {
mMLWildcardDirPathBlacklist.push_back(mExcludeDirs[i]);
continue;
}
bool isWildcardPath
= mExcludeDirs[i].find("*") != string::npos || mExcludeDirs[i].find("?") != string::npos;
if (isWildcardPath) {
mWildcardDirPathBlacklist.push_back(mExcludeDirs[i]);
} else {
mDirPathBlacklist.push_back(mExcludeDirs[i]);
}
}
}
if (!mDirPathBlacklist.empty() || !mWildcardDirPathBlacklist.empty() || !mMLWildcardDirPathBlacklist.empty()
|| !mMLFilePathBlacklist.empty() || !mFileNameBlacklist.empty() || !mFilePathBlacklist.empty()) {
mHasBlacklist = true;
}
// AllowingCollectingFilesInRootDir
if (!GetOptionalBoolParam(
config, "AllowingCollectingFilesInRootDir", mAllowingCollectingFilesInRootDir, errorMsg)) {
PARAM_WARNING_DEFAULT(ctx.GetLogger(),
ctx.GetAlarm(),
errorMsg,
mAllowingCollectingFilesInRootDir,
pluginType,
ctx.GetConfigName(),
ctx.GetProjectName(),
ctx.GetLogstoreName(),
ctx.GetRegion());
} else if (mAllowingCollectingFilesInRootDir) {
BOOL_FLAG(enable_root_path_collection) = mAllowingCollectingFilesInRootDir;
}
// AllowingIncludedByMultiConfigs
if (!GetOptionalBoolParam(config, "AllowingIncludedByMultiConfigs", mAllowingIncludedByMultiConfigs, errorMsg)) {
PARAM_WARNING_DEFAULT(ctx.GetLogger(),
ctx.GetAlarm(),
errorMsg,
mAllowingIncludedByMultiConfigs,
pluginType,
ctx.GetConfigName(),
ctx.GetProjectName(),
ctx.GetLogstoreName(),
ctx.GetRegion());
}
return true;
}