core/unittest/config/ConfigMatchUnittest.cpp (887 lines of code) (raw):
// Copyright 2022 iLogtail Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <assert.h>
#include "unittest/Unittest.h"
#if defined(__linux__)
#include <fnmatch.h>
#include <signal.h>
#include <sys/inotify.h>
#include <unistd.h>
#endif
#include <errno.h>
#include <fcntl.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <fstream>
#include <iostream>
#include <memory>
#include <set>
#include <string>
#include <thread>
#include <typeinfo>
#include "json/json.h"
#include "app_config/AppConfig.h"
#include "common/FileSystemUtil.h"
#include "common/Flags.h"
#include "file_server/ConfigManager.h"
#include "file_server/EventDispatcher.h"
#include "file_server/event/Event.h"
#include "file_server/event_handler/EventHandler.h"
#include "file_server/event_handler/LogInput.h"
#include "file_server/polling/PollingEventQueue.h"
#include "file_server/reader/LogFileReader.h"
#include "logger/Logger.h"
using namespace std;
DECLARE_FLAG_STRING(ilogtail_config);
DECLARE_FLAG_STRING(user_log_config);
DECLARE_FLAG_INT32(batch_send_interval);
namespace logtail {
string gRootDir = "";
int gCaseID = 0;
std::unique_ptr<std::thread> gDispatchThreadId;
void RunningDispatcher() {
ConfigManager::GetInstance()->RegisterHandlers();
EventDispatcher::GetInstance()->Dispatch();
}
class ConfigMatchUnittest : public ::testing::Test {
public:
static void SetUpTestCase() {
gRootDir = GetProcessExecutionDir();
if (gRootDir.at(gRootDir.size() - 1) == PATH_SEPARATOR[0])
gRootDir.resize(gRootDir.size() - 1);
gRootDir += PATH_SEPARATOR + "ilogtailConfigMatchTest";
bfs::remove_all(gRootDir);
bfs::create_directories(gRootDir);
srand(time(NULL));
}
static void TearDownTestCase() { bfs::remove_all(gRootDir); }
void CaseSetUp() {
bfs::remove("log_file_out");
AppConfig::GetInstance()->LoadAppConfig(STRING_FLAG(ilogtail_config));
ConfigManager::GetInstance()->LoadConfig(STRING_FLAG(user_log_config));
gDispatchThreadId.reset(new std::thread(RunningDispatcher));
sleep(1);
}
void CaseCleanUp() {
EventDispatcher::GetInstance()->CleanEnviroments();
gDispatchThreadId->join();
gDispatchThreadId = nullptr;
ConfigManager::GetInstance()->CleanEnviroments();
bfs::remove("user_log_config.json");
bfs::remove("loongcollector_config.json");
bfs::remove_all(gRootDir);
}
string GenerateRandomStr(int32_t minLength, int32_t maxLength) {
int32_t length = (rand() % (maxLength - minLength + 1)) + minLength;
string randomStr = "";
for (int32_t i = 0; i < length; ++i) {
// A-Z: 65-90
int temp = (rand() % (90 - 65 + 1)) + 65;
randomStr += (char)temp;
}
return randomStr;
}
string GenerateDirWithDepth(int32_t minDepth, int32_t maxDepth) {
int32_t depth = (rand() % (maxDepth - minDepth + 1)) + minDepth;
string randomDir = "";
for (int32_t i = 0; i < depth; ++i) {
randomDir += PATH_SEPARATOR + GenerateRandomStr(1, 1);
}
return randomDir;
}
void MockDefaultConfigs() {
Json::Value rootJson;
// commonreg1.com -- logpath: A/B; preserve: false; preserve_depth: 1
Json::Value commonreg_com;
commonreg_com["project_name"] = Json::Value("1000000_proj");
commonreg_com["category"] = Json::Value("app_log");
commonreg_com["log_type"] = Json::Value("common_reg_log");
commonreg_com["log_path"] = Json::Value(gRootDir + "/A/B");
commonreg_com["file_pattern"] = Json::Value("*.[Ll][Oo][Gg]");
commonreg_com["enable"] = Json::Value(true);
commonreg_com["timeformat"] = Json::Value("%d/%b/%Y:%H:%M:%S");
commonreg_com["local_storage"] = Json::Value(true);
// added by xianzhi(bowen.gbw@antfin.com)
Json::Value customizedFieldsValue, dataIntegrityValue, lineCountValue;
dataIntegrityValue["switch"] = Json::Value(true);
dataIntegrityValue["project_name"] = Json::Value("data_integrity");
dataIntegrityValue["switch"] = Json::Value("data_integrity");
dataIntegrityValue["log_time_reg"] = Json::Value("([0-9]{4})-(0[0-9]{1}|1[0-2])-(0[0-9]{1}|[12][0-9]{1}|3[01]) "
"(0[0-9]{1}|1[0-9]{1}|2[0-3]):[0-5][0-9]{1}:([0-5][0-9]{1})");
customizedFieldsValue["data_integrity"] = dataIntegrityValue;
lineCountValue["switch"] = Json::Value(true);
lineCountValue["project_name"] = Json::Value("line_count");
lineCountValue["switch"] = Json::Value("line_count");
customizedFieldsValue["line_count"] = lineCountValue;
commonreg_com["customized"] = customizedFieldsValue;
Json::Value regs;
regs.append(Json::Value("([\\d\\.]+) \\S+ \\S+ \\[(\\S+) \\S+\\] (\\S+) (\\d+)"));
Json::Value keys;
keys.append(Json::Value("ip,time,nothing,seq"));
commonreg_com["regex"] = regs;
commonreg_com["keys"] = keys;
commonreg_com["preserve"] = Json::Value(false);
commonreg_com["preserve_depth"] = Json::Value(1);
rootJson["commonreg1.com"] = commonreg_com;
// commonreg2.com -- logpath: A/B/C/Service; preserve: true
Json::Value commonreg_com2;
commonreg_com2["project_name"] = Json::Value("1000000_proj");
commonreg_com2["category"] = Json::Value("app_log");
commonreg_com2["log_type"] = Json::Value("common_reg_log");
commonreg_com2["log_path"] = Json::Value(gRootDir + "/A/B/C/Service");
commonreg_com2["file_pattern"] = Json::Value("*.[Ll][Oo][Gg]");
commonreg_com2["enable"] = Json::Value(true);
commonreg_com2["timeformat"] = Json::Value("%d/%b/%Y:%H:%M:%S");
commonreg_com2["local_storage"] = Json::Value(true);
Json::Value regs2;
regs2.append(Json::Value("([\\d\\.]+) \\S+ \\S+ \\[(\\S+) \\S+\\] (\\S+) (\\d+)"));
Json::Value keys2;
keys2.append(Json::Value("ip,time,nothing,seq"));
commonreg_com2["regex"] = regs2;
commonreg_com2["keys"] = keys2;
commonreg_com2["preserve"] = Json::Value(true);
rootJson["commonreg2.com"] = commonreg_com2;
Json::Value secondary;
secondary["max_flow_byte_per_sec"] = Json::Value("10240000");
secondary["max_num_of_file"] = Json::Value("10");
secondary["enable_secondady"] = Json::Value(true);
Json::Value metrics;
metrics["metrics"] = rootJson;
metrics["local_storage"] = secondary;
ofstream fout("user_log_config.json");
fout << metrics << endl;
fout.close();
}
void MockFoceMultiConfigs() {
Json::Value rootJson;
// commonreg1.com -- logpath: A/B; preserve: false; preserve_depth: 1
Json::Value commonreg_com;
commonreg_com["project_name"] = Json::Value("1000000_proj");
commonreg_com["category"] = Json::Value("app_log");
commonreg_com["log_type"] = Json::Value("common_reg_log");
commonreg_com["log_path"] = Json::Value(gRootDir + PATH_SEPARATOR + "A" + PATH_SEPARATOR + "B");
#if defined(_MSC_VER)
commonreg_com["file_pattern"] = Json::Value("*.log");
#else
commonreg_com["file_pattern"] = Json::Value("*.[Ll][Oo][Gg]");
#endif
commonreg_com["enable"] = Json::Value(true);
commonreg_com["timeformat"] = Json::Value("%d/%b/%Y:%H:%M:%S");
commonreg_com["local_storage"] = Json::Value(true);
Json::Value regs;
regs.append(Json::Value("([\\d\\.]+) \\S+ \\S+ \\[(\\S+) \\S+\\] (\\S+) (\\d+)"));
Json::Value keys;
keys.append(Json::Value("ip,time,nothing,seq"));
commonreg_com["regex"] = regs;
commonreg_com["keys"] = keys;
commonreg_com["preserve"] = Json::Value(false);
commonreg_com["preserve_depth"] = Json::Value(1);
rootJson["commonreg1.com"] = commonreg_com;
Json::Value commonreg_com2;
commonreg_com2 = commonreg_com;
Json::Value advancedConfig;
advancedConfig["force_multiconfig"] = Json::Value(false);
commonreg_com2["advanced"] = advancedConfig;
rootJson["commonreg2.com"] = commonreg_com2;
Json::Value commonreg_com3;
commonreg_com3 = commonreg_com;
advancedConfig["force_multiconfig"] = Json::Value(true);
commonreg_com3["advanced"] = advancedConfig;
rootJson["commonreg3.com"] = commonreg_com3;
Json::Value commonreg_com4;
commonreg_com4 = commonreg_com;
commonreg_com4["log_path"] = Json::Value("/no_this_path");
rootJson["commonreg4.com"] = commonreg_com4;
Json::Value metrics;
metrics["metrics"] = rootJson;
ofstream fout("user_log_config.json");
fout << metrics << endl;
fout.close();
}
void TestForceMultiConfig() {
LOG_INFO(sLogger, ("TestForceMultiConfig() begin", time(NULL)));
MockFoceMultiConfigs();
auto& PS = PATH_SEPARATOR;
CaseSetUp();
{
vector<FileDiscoveryConfig> allConfig;
ConfigManager::GetInstance()->FindMatchWithForceFlag(allConfig, gRootDir + PS + "A" + PS + "B", "");
APSARA_TEST_EQUAL(allConfig.size(), (size_t)2);
}
{
vector<FileDiscoveryConfig> allConfig;
ConfigManager::GetInstance()->FindMatchWithForceFlag(allConfig, gRootDir + PS + "A" + PS + "B", "test.Log");
APSARA_TEST_EQUAL(allConfig.size(), (size_t)2);
}
ConfigManager::GetInstance()->mCacheFileAllConfigMap.clear();
{
vector<FileDiscoveryConfig> allConfig;
ConfigManager::GetInstance()->FindAllMatch(allConfig, gRootDir + PS + "A" + PS + "B", "test.Log");
APSARA_TEST_EQUAL(allConfig.size(), (size_t)3);
}
{
vector<FileDiscoveryConfig> allConfig;
ConfigManager::GetInstance()->FindMatchWithForceFlag(allConfig, gRootDir + PS, "");
APSARA_TEST_EQUAL(allConfig.size(), (size_t)0);
}
LOG_INFO(sLogger, ("TestForceMultiConfig() done", time(NULL)));
CaseCleanUp();
}
void TestConfigLongestMatch() {
LOG_INFO(sLogger, ("TestConfigLongestMatch() begin", time(NULL)));
// TODO: Add for Windows.
#if defined(__linux__)
MockDefaultConfigs();
CaseSetUp();
string path[] = {
// timeout
"/A/B", // false
"/A/B/Job", // false
"/A/B/C", // false
"/A/B/C/Job", // true
"/A/B/C/Service", // false
"/A/B/C/Service/s1", // false
"/A/B/C/Service/s2" // false
};
bool result[] = {false, false, false, true, false, false, false};
bfs::create_directories(gRootDir + path[0]);
bfs::create_directories(gRootDir + path[4]);
sleep(65);
for (size_t i = 1; i < 7; i++) {
if (i == 4)
continue;
bfs::create_directories(gRootDir + path[i]);
}
sleep(1);
for (size_t i = 0; i < 7; i++) {
string dir = gRootDir + path[i];
std::unordered_map<std::string, int>::iterator pwItr
= (EventDispatcher::GetInstance()->mPathWdMap).find(dir);
APSARA_TEST_TRUE_DESC(pwItr != EventDispatcher::GetInstance()->mPathWdMap.end(), dir);
if (pwItr != EventDispatcher::GetInstance()->mPathWdMap.end()) {
unordered_map<int, time_t>::iterator wuItr
= EventDispatcher::GetInstance()->mWdUpdateTimeMap.find(pwItr->second);
APSARA_TEST_EQUAL_DESC(wuItr != EventDispatcher::GetInstance()->mWdUpdateTimeMap.end(), result[i], dir);
}
}
CaseCleanUp();
#endif
LOG_INFO(sLogger, ("TestConfigLongestMatch() end", time(NULL)));
}
void MockLotsOfConfigs() {
Json::Value rootJson;
// common regex log
for (size_t i = 0; i < 1000; i++) {
string logName = "common_regex_log_" + ToString(i);
Json::Value commonreg_com;
commonreg_com["project_name"] = Json::Value("1000000_proj");
commonreg_com["category"] = Json::Value("app_log");
commonreg_com["log_type"] = Json::Value("common_reg_log");
commonreg_com["log_path"] = Json::Value(gRootDir + "/common_reg_log" + GenerateDirWithDepth(1, 4));
commonreg_com["file_pattern"] = Json::Value("*.[Ll][Oo][Gg]");
commonreg_com["enable"] = Json::Value(true);
commonreg_com["timeformat"] = Json::Value("%d/%b/%Y:%H:%M:%S");
commonreg_com["local_storage"] = Json::Value(true);
Json::Value regs;
regs.append(Json::Value("([\\d\\.]+) \\S+ \\S+ \\[(\\S+) \\S+\\] (\\S+) (\\d+)"));
Json::Value keys;
keys.append(Json::Value("ip,time,nothing,seq"));
commonreg_com["regex"] = regs;
commonreg_com["keys"] = keys;
commonreg_com["preserve"] = Json::Value(false);
commonreg_com["preserve_depth"] = Json::Value(1);
rootJson[logName] = commonreg_com;
}
// apsara log
for (size_t i = 0; i < 1000; ++i) {
string logName = "apsara_log_" + ToString(i);
Json::Value apsara_log;
apsara_log["project_name"] = Json::Value("9000000_proj");
apsara_log["category"] = Json::Value(string("9000000_category_") + ToString(i));
apsara_log["log_type"] = Json::Value("apsara_log");
apsara_log["log_begin_reg"] = Json::Value("\\[\\d+-\\d+-\\d+ \\d+:\\d+:\\d+.\\d+\\].*");
apsara_log["log_path"] = Json::Value(gRootDir + "/apsara_log" + GenerateDirWithDepth(1, 4));
apsara_log["file_pattern"] = Json::Value("*.[Ll][Oo][Gg]");
apsara_log["enable"] = Json::Value(true);
apsara_log["preserve"] = Json::Value(true);
apsara_log["topic_format"] = Json::Value("default");
rootJson[logName] = apsara_log;
}
Json::Value secondary;
secondary["max_flow_byte_per_sec"] = Json::Value("10240000");
secondary["max_num_of_file"] = Json::Value("10");
secondary["enable_secondady"] = Json::Value(true);
Json::Value metrics;
metrics["metrics"] = rootJson;
metrics["local_storage"] = secondary;
ofstream fout("user_log_config.json");
fout << metrics << endl;
fout.close();
}
void TestLotsOfConfigs() {
LOG_INFO(sLogger, ("TestLotsOfConfigs() begin", time(NULL)));
MockLotsOfConfigs();
CaseSetUp();
sleep(1);
APSARA_TEST_EQUAL(int(ConfigManager::GetInstance()->mNameConfigMap.size()), 2000);
CaseCleanUp();
LOG_INFO(sLogger, ("TestLotsOfConfigs() end", time(NULL)));
}
void MockMaxDepthConfigs() {
Json::Value rootJson;
// commonreg1.com
Json::Value commonreg_com;
commonreg_com["project_name"] = Json::Value("1000000_proj");
commonreg_com["category"] = Json::Value("app_log");
commonreg_com["log_type"] = Json::Value("common_reg_log");
commonreg_com["log_path"] = Json::Value(gRootDir + PATH_SEPARATOR + "A");
commonreg_com["file_pattern"] = Json::Value("*.[Ll][Oo][Gg]");
commonreg_com["enable"] = Json::Value(true);
commonreg_com["timeformat"] = Json::Value("%d/%b/%Y:%H:%M:%S");
commonreg_com["local_storage"] = Json::Value(true);
Json::Value regs;
regs.append(Json::Value("([\\d\\.]+) \\S+ \\S+ \\[(\\S+) \\S+\\] (\\S+) (\\d+)"));
Json::Value keys;
keys.append(Json::Value("ip,time,nothing,seq"));
commonreg_com["regex"] = regs;
commonreg_com["keys"] = keys;
commonreg_com["preserve"] = Json::Value(true);
commonreg_com["max_depth"] = -1; // default
rootJson["commonreg1.com"] = commonreg_com;
// commonreg2.com
Json::Value commonreg_com2;
commonreg_com2["project_name"] = Json::Value("1000000_proj");
commonreg_com2["category"] = Json::Value("app_log_2");
commonreg_com2["log_type"] = Json::Value("common_reg_log");
commonreg_com2["log_path"] = Json::Value(gRootDir + PATH_SEPARATOR + "B");
commonreg_com2["file_pattern"] = Json::Value("*.[Ll][Oo][Gg]");
commonreg_com2["enable"] = Json::Value(true);
commonreg_com2["timeformat"] = Json::Value("%d/%b/%Y:%H:%M:%S");
commonreg_com2["local_storage"] = Json::Value(true);
Json::Value regs2;
regs2.append(Json::Value("([\\d\\.]+) \\S+ \\S+ \\[(\\S+) \\S+\\] (\\S+) (\\d+)"));
Json::Value keys2;
keys2.append(Json::Value("ip,time,nothing,seq"));
commonreg_com2["regex"] = regs2;
commonreg_com2["keys"] = keys2;
commonreg_com2["preserve"] = Json::Value(true);
commonreg_com2["max_depth"] = 0;
rootJson["commonreg2.com"] = commonreg_com2;
// commonreg3.com
Json::Value commonreg_com3;
commonreg_com3["project_name"] = Json::Value("1000000_proj");
commonreg_com3["category"] = Json::Value("app_log_3");
commonreg_com3["log_type"] = Json::Value("common_reg_log");
commonreg_com3["log_path"] = Json::Value(gRootDir + PATH_SEPARATOR + "C");
commonreg_com3["file_pattern"] = Json::Value("*.[Ll][Oo][Gg]");
commonreg_com3["enable"] = Json::Value(true);
commonreg_com3["timeformat"] = Json::Value("%d/%b/%Y:%H:%M:%S");
commonreg_com3["local_storage"] = Json::Value(true);
Json::Value regs3;
regs3.append(Json::Value("([\\d\\.]+) \\S+ \\S+ \\[(\\S+) \\S+\\] (\\S+) (\\d+)"));
Json::Value keys3;
keys3.append(Json::Value("ip,time,nothing,seq"));
commonreg_com3["regex"] = regs2;
commonreg_com3["keys"] = keys2;
commonreg_com3["preserve"] = Json::Value(true);
commonreg_com3["max_depth"] = 2;
rootJson["commonreg3.com"] = commonreg_com3;
Json::Value secondary;
secondary["max_flow_byte_per_sec"] = Json::Value("10240000");
secondary["max_num_of_file"] = Json::Value("10");
secondary["enable_secondady"] = Json::Value(true);
Json::Value metrics;
metrics["metrics"] = rootJson;
metrics["local_storage"] = secondary;
ofstream fout("user_log_config.json");
fout << metrics << endl;
fout.close();
}
void TestMaxWatchDepth() {
LOG_INFO(sLogger, ("TestMaxWatchDepth() begin", time(NULL)));
auto const& PS = PATH_SEPARATOR;
string path[12] = {PS + "A",
PS + "B",
PS + "C",
PS + "A" + PS + "a",
PS + "A" + PS + "a" + PS + "aa",
PS + "A" + PS + "a" + PS + "aa" + PS + "aaa",
PS + "A" + PS + "a" + PS + "aa" + PS + "aaa" + PS + "aaaa",
PS + "B" + PS + "b",
PS + "B" + PS + "bb",
PS + "C" + PS + "c",
PS + "C" + PS + "c" + PS + "cc",
PS + "C" + PS + "c" + PS + "cc" + PS + "ccc"};
bool result[12] = {true, true, true, true, true, true, true, false, false, true, true, false};
LOG_INFO(sLogger, ("case 1", "child directory create dynamically"));
MockMaxDepthConfigs();
int32_t idx = 0;
for (; idx < 3; idx++) {
string cmd("mkdir -p ");
cmd.append(gRootDir).append(path[idx]);
bfs::create_directories(bfs::path(gRootDir) / path[idx]);
LOG_INFO(sLogger, ("cmd", cmd));
}
CaseSetUp();
for (; idx < 12; idx++) {
string cmd("mkdir -p ");
cmd.append(gRootDir).append(path[idx]);
bfs::create_directories(bfs::path(gRootDir) / path[idx]);
LOG_INFO(sLogger, ("cmd", cmd));
usleep(300 * 1000);
}
sleep(3);
#if defined(_MSC_VER)
// On Windows, only polling is available, so we have to wait for a longer time.
sleep(LogInput::GetInstance()->mCheckBaseDirInterval);
#endif
for (idx = 0; idx < 12; ++idx) {
string dir = gRootDir + path[idx];
bool registered = EventDispatcher::GetInstance()->mPathWdMap.find(dir)
!= EventDispatcher::GetInstance()->mPathWdMap.end();
APSARA_TEST_EQUAL_DESC(registered, result[idx], dir);
}
int32_t bakInterval = LogInput::GetInstance()->mCheckBaseDirInterval;
LogInput::GetInstance()->mCheckBaseDirInterval = 86400;
sleep(1);
for (idx = 0; idx < 3; idx++) {
EventDispatcher::GetInstance()->UnregisterAllDir(gRootDir + path[idx]);
LOG_INFO(sLogger, ("UnregisterAllDir", gRootDir + path[idx]));
}
LOG_INFO(sLogger, ("case 2", "UnregisterAllDir"));
sleep(1);
for (idx = 0; idx < 12; ++idx) {
string dir = gRootDir + path[idx];
bool registered = EventDispatcher::GetInstance()->mPathWdMap.find(dir)
== EventDispatcher::GetInstance()->mPathWdMap.end();
APSARA_TEST_TRUE_DESC(registered, dir);
}
CaseCleanUp();
sleep(3);
LOG_INFO(sLogger, ("case 3", "directory already exists"));
for (idx = 0; idx < 12; idx++) {
string cmd("mkdir -p ");
cmd.append(gRootDir).append(path[idx]);
bfs::create_directories(bfs::path(gRootDir) / path[idx]);
LOG_INFO(sLogger, ("cmd", cmd));
}
MockMaxDepthConfigs();
CaseSetUp();
sleep(1);
for (idx = 0; idx < 12; ++idx) {
string dir = gRootDir + path[idx];
bool registered = EventDispatcher::GetInstance()->mPathWdMap.find(dir)
!= EventDispatcher::GetInstance()->mPathWdMap.end();
APSARA_TEST_EQUAL_DESC(registered, result[idx], dir);
}
LogInput::GetInstance()->mCheckBaseDirInterval = bakInterval;
CaseCleanUp();
LOG_INFO(sLogger, ("TestMaxWatchDepth() end", time(NULL)));
}
void TestChinesePathAndFilePattern();
void TestBlacklistControlInIsMatch();
void TestBlacklistControl();
void TestBlacklistControlWildcardBasePath();
private:
void GenerateUserLogConfigForTestingBlacklistControl(const std::string& pathRoot, const std::string& logPath = "");
void TestBlacklistControlCommon(const std::string& pathRoot, const std::string& logPath = "") {
GenerateUserLogConfigForTestingBlacklistControl(pathRoot);
std::ofstream out("loongcollector_config.json");
out << std::string("{") << std::string("\"config_server_address\" : \"file\",")
<< std::string("\"data_server_address\" : \"file\",") << std::string("\"domain\" : \"\"")
<< std::string("}");
out.close();
CaseSetUp();
// Generate log for each case.
auto GenerateLog = [](const bfs::path& logPath) {
bfs::create_directories(bfs::path(logPath).parent_path());
static int32_t lastLogID = 0;
std::ofstream(logPath.string(), std::ios_base::app) << "LOGID:" << lastLogID << std::endl;
return lastLogID++;
};
std::vector<int32_t> acceptedLogIDs;
// Rule 1 in directory blacklist.
GenerateLog(pathRoot + "/1" + "/1.log");
GenerateLog(pathRoot + "/1" + "/2.log");
GenerateLog(pathRoot + "/1" + "/2" + "/3.log");
GenerateLog(pathRoot + "/1" + "/3" + "/4" + "/4.log");
// Rule 2 in directory blacklist.
GenerateLog(pathRoot + "/4" + "/1" + "/1.log");
GenerateLog(pathRoot + "/4" + "/1" + "/1" + "/1.log");
acceptedLogIDs.push_back(GenerateLog(pathRoot + "/4" + "/1.log"));
// Rule 3 in directory blacklist.
GenerateLog(pathRoot + "/5" + "/a" + "/1" + "/1.log");
GenerateLog(pathRoot + "/5" + "/a" + "/1" + "/2" + "/1.log");
acceptedLogIDs.push_back(GenerateLog(pathRoot + "/5" + "/1" + "/1.log"));
#if defined(_MSC_VER)
GenerateLog(pathRoot + "/5" + "/a" + "/b" + "/1" + "/1.log");
#else
acceptedLogIDs.push_back(GenerateLog(pathRoot + "/5" + "/a" + "/b" + "/1" + "/1.log"));
#endif
// Rule 4 in directory blacklist.
GenerateLog(pathRoot + "/6" + "/a" + "/b" + "/1" + "/1.log");
GenerateLog(pathRoot + "/6" + "/a" + "/b" + "/1" + "/2" + "/1.log");
acceptedLogIDs.push_back(GenerateLog(pathRoot + "/6" + "/1" + "/1.log"));
acceptedLogIDs.push_back(GenerateLog(pathRoot + "/6" + "/a" + "/1" + "/1.log"));
#if defined(_MSC_VER)
GenerateLog(pathRoot + "/6" + "/a" + "/b" + "/c" + "/1" + "/1.log");
#else
acceptedLogIDs.push_back(GenerateLog(pathRoot + "/6" + "/a" + "/b" + "/c" + "/1" + "/1.log"));
#endif
// Rule 5 in directory blacklist.
GenerateLog(pathRoot + "/10" + "/a" + "/1.log");
GenerateLog(pathRoot + "/10" + "/a" + "/b" + "/1" + "/1.log");
acceptedLogIDs.push_back(GenerateLog(pathRoot + "/10" + "/1.log"));
// Rule 6 in directory blacklist.
GenerateLog(pathRoot + "/11" + "/a" + "/1" + "/1.log");
GenerateLog(pathRoot + "/11" + "/a" + "/b" + "/1" + "/1.log");
acceptedLogIDs.push_back(GenerateLog(pathRoot + "/11" + "/1" + "/1.log"));
acceptedLogIDs.push_back(GenerateLog(pathRoot + "/11" + "/2" + "/1.log"));
acceptedLogIDs.push_back(GenerateLog(pathRoot + "/11" + "/a" + "/2" + "/1.log"));
acceptedLogIDs.push_back(GenerateLog(pathRoot + "/11" + "/a" + "/b" + "/3" + "/1.log"));
// Rule 1 in file path blacklist.
GenerateLog(pathRoot + "/2" + "/2" + "/ignore.log");
acceptedLogIDs.push_back(GenerateLog(pathRoot + "/2" + "/2" + "/1.log"));
acceptedLogIDs.push_back(GenerateLog(pathRoot + "/2" + "/2" + "/2.log"));
// Rule 2 in file path blacklist.
GenerateLog(pathRoot + "/2" + "/3" + "/ignore.log");
GenerateLog(pathRoot + "/2" + "/3" + "/ignore100.log");
GenerateLog(pathRoot + "/2" + "/3" + "/ignore100_xx.log");
acceptedLogIDs.push_back(GenerateLog((pathRoot + "/2" + "/3" + "/lignore100_xx.log")));
// Rule 3 in file path blacklist.
GenerateLog(pathRoot + "/7" + "/3" + "/ignore.log");
GenerateLog(pathRoot + "/7" + "/4" + "/ignore100.log");
acceptedLogIDs.push_back(GenerateLog(pathRoot + "/7" + "/ignore.log"));
#if defined(_MSC_VER)
GenerateLog(pathRoot + "/7" + "/4" + "/a" + "/ignore.log");
#else
acceptedLogIDs.push_back(GenerateLog(pathRoot + "/7" + "/4" + "/a" + "/ignore.log"));
#endif
// Rule 4 in file path blacklist.
GenerateLog(pathRoot + "/8" + "/3" + "/1" + "/ignore.log");
GenerateLog(pathRoot + "/8" + "/b" + "/1" + "/ignore100.log");
acceptedLogIDs.push_back(GenerateLog(pathRoot + "/8" + "/1" + "/ignore.log"));
#if defined(_MSC_VER)
GenerateLog(pathRoot + "/8" + "/a" + "/b" + "/1" + "/ignore.log");
#else
acceptedLogIDs.push_back(GenerateLog(pathRoot + "/8" + "/a" + "/b" + "/1" + "/ignore.log"));
#endif
// Rule 5 in file path blacklist.
GenerateLog(pathRoot + "/9" + "/a" + "/b" + "/1" + "/ignore.log");
GenerateLog(pathRoot + "/9" + "/a" + "/d" + "/1" + "/ignore100.log");
acceptedLogIDs.push_back(GenerateLog(pathRoot + "/9" + "/1" + "/ignore.log"));
acceptedLogIDs.push_back(GenerateLog(pathRoot + "/9" + "/a" + "/1" + "/ignore.log"));
#if defined(_MSC_VER)
GenerateLog(pathRoot + "/9" + "/a" + "/b" + "/c" + "/1" + "/ignore.log");
#else
acceptedLogIDs.push_back(GenerateLog(pathRoot + "/9" + "/a" + "/b" + "/c" + "/1" + "/ignore.log"));
#endif
// Rule 6 in file path blacklist.
GenerateLog(pathRoot + "/12" + "/a" + "/ignore.log");
GenerateLog(pathRoot + "/12" + "/a" + "/b" + "/c" + "/ignore100.log");
acceptedLogIDs.push_back(GenerateLog(pathRoot + "/12" + "/ignore.log"));
// Rule 7 in file path blacklist.
GenerateLog(pathRoot + "/13" + "/a" + "/1" + "/ignore.log");
GenerateLog(pathRoot + "/13" + "/a" + "/b" + "/1" + "/ignore100.log");
acceptedLogIDs.push_back(GenerateLog(pathRoot + "/13" + "/1" + "/ignore.log"));
acceptedLogIDs.push_back(GenerateLog(pathRoot + "/13" + "/2" + "/ignore.log"));
acceptedLogIDs.push_back(GenerateLog(pathRoot + "/13" + "/a" + "/2" + "/ignore.log"));
acceptedLogIDs.push_back(GenerateLog(pathRoot + "/13" + "/a" + "/b" + "/3" + "/ignore.log"));
// Rule 1 in file name blacklist.
GenerateLog((pathRoot + "/ignoreEverywhere.log"));
GenerateLog((pathRoot + "/ignoreEverywhere100.log"));
GenerateLog((pathRoot + "/1" + "/ignoreEverywhere200.log"));
GenerateLog((pathRoot + "/2" + "/ignoreEverywhere100_xx.log"));
GenerateLog((pathRoot + "/2" + "/3" + "/ignoreEverywhere100_xxyy.log"));
// Normal logs.
acceptedLogIDs.push_back(GenerateLog((pathRoot + "/lignore100_xx.log")));
acceptedLogIDs.push_back(GenerateLog((pathRoot + "/lignore100_xx.log")));
acceptedLogIDs.push_back(GenerateLog((pathRoot + "/2.log")));
acceptedLogIDs.push_back(GenerateLog((pathRoot + "/2.log")));
// Wait for a while, then parse log_file_out.
sleep(3 * INT32_FLAG(batch_send_interval) + 1);
std::vector<int32_t> collectedLogIDs;
std::ifstream in("log_file_out");
std::string line;
while (std::getline(in, line)) {
size_t pos = line.find("LOGID:");
if (std::string::npos == pos) {
continue;
}
int32_t logid{};
StringTo(SplitString(line.substr(line.find("LOGID:")), ":")[1], logid);
collectedLogIDs.push_back(logid);
}
APSARA_TEST_EQUAL(acceptedLogIDs.size(), collectedLogIDs.size());
std::sort(collectedLogIDs.begin(), collectedLogIDs.end());
EXPECT_EQ(acceptedLogIDs, collectedLogIDs);
CaseCleanUp();
}
};
TEST_F(ConfigMatchUnittest, TestConfigLongestMatchWrapper) {
TestConfigLongestMatch();
}
TEST_F(ConfigMatchUnittest, TestLotsOfConfigsWrapper) {
TestLotsOfConfigs();
}
TEST_F(ConfigMatchUnittest, TestMaxWatchDepth) {
TestMaxWatchDepth();
}
APSARA_UNIT_TEST_CASE(ConfigMatchUnittest, TestForceMultiConfig, 0);
// @pathRoot: used to set blacklist rule, if @logPath is empty, set log_path to it.
// @logPath: set log_path.
void ConfigMatchUnittest::GenerateUserLogConfigForTestingBlacklistControl(const std::string& pathRoot,
const std::string& logPath) {
const std::string PS = PATH_SEPARATOR;
Json::Value configJSON;
configJSON["project_name"] = Json::Value("test_project");
configJSON["category"] = Json::Value("test_category");
configJSON["log_type"] = Json::Value("common_reg_log");
configJSON["log_begin_reg"] = Json::Value(".*");
configJSON["log_path"] = Json::Value(logPath.empty() ? pathRoot : logPath);
configJSON["file_pattern"] = "*.log";
configJSON["enable"] = Json::Value(true);
configJSON["preserve"] = Json::Value(true);
configJSON["preserve_depth"] = Json::Value(0);
configJSON["topic_format"] = Json::Value("none");
configJSON["timeformat"] = Json::Value("%d/%b/%Y:%H:%M:%S");
configJSON["version"] = Json::Value(1);
Json::Value keys;
keys.append(Json::Value("content"));
configJSON["keys"] = keys;
Json::Value regs;
regs.append(Json::Value("(.*)"));
configJSON["regex"] = regs;
Json::Value advancedConfig;
Json::Value blacklist;
Json::Value dirBlacklist;
dirBlacklist.append(Json::Value(pathRoot + PS + "1"));
dirBlacklist.append(Json::Value(pathRoot + PS + "4" + PS + "*"));
dirBlacklist.append(Json::Value(pathRoot + PS + "5" + PS + "*" + PS + "1"));
dirBlacklist.append(Json::Value(pathRoot + PS + "6" + PS + "*" + PS + "*" + PS + "1"));
dirBlacklist.append(Json::Value(pathRoot + PS + "10" + PS + "**"));
dirBlacklist.append(Json::Value(pathRoot + PS + "11" + PS + "**" + PS + "1"));
blacklist["dir_blacklist"] = dirBlacklist;
Json::Value fileNameBlacklist;
fileNameBlacklist.append(Json::Value("ignoreEverywhere*.log"));
blacklist["filename_blacklist"] = fileNameBlacklist;
Json::Value filePathBlacklist;
filePathBlacklist.append(Json::Value(pathRoot + PS + "2" + PS + "2" + PS + "ignore.log"));
filePathBlacklist.append(Json::Value(pathRoot + PS + "2" + PS + "3" + PS + "ignore*.log"));
filePathBlacklist.append(Json::Value(pathRoot + PS + "7" + PS + "*" + PS + "ignore*.log"));
filePathBlacklist.append(Json::Value(pathRoot + PS + "8" + PS + "*" + PS + "1" + PS + "ignore*.log"));
filePathBlacklist.append(Json::Value(pathRoot + PS + "9" + PS + "*" + PS + "*" + PS + "1" + PS + "ignore*.log"));
filePathBlacklist.append(Json::Value(pathRoot + PS + "12" + PS + "**" + PS + "ignore*.log"));
filePathBlacklist.append(Json::Value(pathRoot + PS + "13" + PS + "**" + PS + "1" + PS + "ignore*.log"));
blacklist["filepath_blacklist"] = filePathBlacklist;
advancedConfig["blacklist"] = blacklist;
configJSON["advanced"] = advancedConfig;
Json::Value rootJSON, metricsJSON;
rootJSON["common_reg_log"] = configJSON;
metricsJSON["metrics"] = rootJSON;
std::ofstream out;
out.open(STRING_FLAG(user_log_config).c_str());
out << metricsJSON.toStyledString() << std::endl;
}
// File not in docker and with normal base path.
void ConfigMatchUnittest::TestBlacklistControlInIsMatch() {
LOG_INFO(sLogger, ("TestBlacklistControlInIsMatch() begin", time(NULL)));
auto& PS = PATH_SEPARATOR;
auto& pathRoot = gRootDir;
GenerateUserLogConfigForTestingBlacklistControl(pathRoot);
CaseSetUp();
auto configManagerInst = ConfigManager::GetInstance();
auto& configMap = configManagerInst->GetAllConfig();
ASSERT_EQ(configMap.size(), 1);
auto beginIter = configMap.begin();
EXPECT_EQ(beginIter->first, "common_reg_log");
auto& config = *(beginIter->second);
EXPECT_EQ(config.mDirPathBlacklist.size(), 1);
EXPECT_EQ(config.mWildcardDirPathBlacklist.size(), 3);
EXPECT_EQ(config.mMLWildcardDirPathBlacklist.size(), 2);
EXPECT_EQ(config.mFilePathBlacklist.size(), 5);
EXPECT_EQ(config.mMLFilePathBlacklist.size(), 2);
EXPECT_EQ(config.mFileNameBlacklist.size(), 1);
struct Case {
std::string path;
std::string name;
bool expected;
std::string String() const {
std::ostringstream oss;
oss << "Case{\n"
<< "\tPath:" << path << "\n"
<< "\tName:" << name << "\n"
<< "\tExpected:" << expected << "\n"
<< "}";
return oss.str();
}
};
std::vector<Case> blacklistedCases;
#define ADD_CASE(dirPath, fileName) \
do { \
blacklistedCases.push_back({dirPath, fileName, false}); \
if (std::string(fileName).empty()) \
blacklistedCases.push_back({dirPath, "1.log", false}); \
} while (0);
// Rule 1 in directory blacklist.
ADD_CASE(pathRoot + PS + "1", "");
ADD_CASE(pathRoot + PS + "1" + PS + "2", "");
ADD_CASE(pathRoot + PS + "1" + PS + "3" + PS + "4", "");
// Rule 2 in directory blacklist.
ADD_CASE(pathRoot + PS + "4" + PS + "1", "");
// It's insignificant that IsMatch can match following two cases, because if their
// parent directory is filtered, they will not be discovered, no IsMatch will
// be called on them.
// - {pathRoot + PS + "4" + PS + "1" + PS + "1", "", true}
// - {pathRoot + PS + "4" + PS + "1" + PS + "1", "3.log", true}
// Rule 3 in directory blacklist.
ADD_CASE(pathRoot + PS + "5" + PS + "a" + PS + "1", "");
// Rule 4 in directory blacklist.
ADD_CASE(pathRoot + PS + "6" + PS + "a" + PS + "b" + PS + "1", "");
// Rule 5 in directory blacklist.
ADD_CASE(pathRoot + PS + "10" + PS + "a", "");
ADD_CASE(pathRoot + PS + "10" + PS + "a" + PS + "b" + PS + "1", "");
// Rule 6 in directory blacklist.
ADD_CASE(pathRoot + PS + "11" + PS + "a" + PS + "1", "");
ADD_CASE(pathRoot + PS + "11" + PS + "a" + PS + "b" + PS + "1", "");
// Rule 1 in file path blacklist.
ADD_CASE(pathRoot + PS + "2" + PS + "2", "ignore.log");
// Rule 2 in file path blacklist.
ADD_CASE(pathRoot + PS + "2" + PS + "3", "ignore.log");
ADD_CASE(pathRoot + PS + "2" + PS + "3", "ignore100.log");
ADD_CASE(pathRoot + PS + "2" + PS + "3", "ignore100_xx.log");
// Rule 3 in file path blacklist.
ADD_CASE(pathRoot + PS + "7" + PS + "3", "ignore.log");
ADD_CASE(pathRoot + PS + "7" + PS + "4", "ignore100.log");
// Rule 4 in file path blacklist.
ADD_CASE(pathRoot + PS + "8" + PS + "3" + PS + "1", "ignore.log");
ADD_CASE(pathRoot + PS + "8" + PS + "b" + PS + "1", "ignore100.log");
// Rule 5 in file path blacklist.
ADD_CASE(pathRoot + PS + "9" + PS + "a" + PS + "b" + PS + "1", "ignore.log");
ADD_CASE(pathRoot + PS + "9" + PS + "a" + PS + "d" + PS + "1", "ignore100.log");
// Rule 6 in file path blacklist.
ADD_CASE(pathRoot + PS + "12" + PS + "a", "ignore.log");
ADD_CASE(pathRoot + PS + "12" + PS + "a" + PS + "b" + PS + "c", "ignore100.log");
// Rule 7 in file path blacklist.
ADD_CASE(pathRoot + PS + "13" + PS + "a" + PS + "1", "ignore.log");
ADD_CASE(pathRoot + PS + "13" + PS + "a" + PS + "b" + PS + "1", "ignore100.log");
// Rule 1 in file name blacklist.
ADD_CASE(pathRoot, "ignoreEverywhere.log");
ADD_CASE(pathRoot + PS + "3" + PS + "3", "ignoreEverywhere100.log");
ADD_CASE(pathRoot + PS + "4" + PS + "5", "ignoreEverywhere100.log");
for (size_t i = 0; i < blacklistedCases.size(); ++i) {
auto& c = blacklistedCases[i];
auto result = config.IsMatch(c.path, c.name);
LOG_INFO(sLogger, ("Blacklist case", i)("detail", c.String())("actual", result));
APSARA_TEST_EQUAL_DESC(c.expected, result, c.String());
}
#undef ADD_CASE
std::vector<Case> acceptedCases;
std::string tmpPath;
#define ADD_CASE(dirPath, fileName) \
do { \
acceptedCases.push_back({dirPath, fileName, true}); \
if (std::string(fileName).empty()) \
acceptedCases.push_back({dirPath, "1.log", true}); \
} while (0);
// Won't block by rule 2 in directory blacklist.
tmpPath = pathRoot + PS + "4";
APSARA_TEST_EQUAL(fnmatch((pathRoot + PS + "4" + PS + "*").c_str(), tmpPath.c_str(), FNM_PATHNAME), 1);
ADD_CASE(tmpPath, "");
// Won't block by rule 3 in directory blacklist.
tmpPath = pathRoot + PS + "5" + PS + "1";
APSARA_TEST_EQUAL(fnmatch((pathRoot + PS + "5" + PS + "*" + PS + "1").c_str(), tmpPath.c_str(), FNM_PATHNAME), 1);
ADD_CASE(tmpPath, "");
tmpPath = pathRoot + PS + "5" + PS + "a" + PS + "b" + PS + "1";
#if defined(_MSC_VER)
APSARA_TEST_EQUAL(fnmatch((pathRoot + PS + "5" + PS + "*" + PS + "1").c_str(), tmpPath.c_str(), FNM_PATHNAME), 0);
#else
APSARA_TEST_EQUAL(fnmatch((pathRoot + PS + "5" + PS + "*" + PS + "1").c_str(), tmpPath.c_str(), FNM_PATHNAME), 1);
ADD_CASE(tmpPath, "");
#endif
// Won't block by rule 4 in directory blacklist.
tmpPath = pathRoot + PS + "6" + PS + "1";
APSARA_TEST_EQUAL(
fnmatch((pathRoot + PS + "6" + PS + "*" + PS + "*" + PS + "1").c_str(), tmpPath.c_str(), FNM_PATHNAME), 1);
ADD_CASE(tmpPath, "");
tmpPath = pathRoot + PS + "6" + PS + "a" + PS + "1";
APSARA_TEST_EQUAL(
fnmatch((pathRoot + PS + "6" + PS + "*" + PS + "*" + PS + "1").c_str(), tmpPath.c_str(), FNM_PATHNAME), 1);
ADD_CASE(tmpPath, "");
tmpPath = pathRoot + PS + "6" + PS + "a" + PS + "b" + PS + "c" + PS + "1";
#if defined(_MSC_VER)
APSARA_TEST_EQUAL(
fnmatch((pathRoot + PS + "6" + PS + "*" + PS + "*" + PS + "1").c_str(), tmpPath.c_str(), FNM_PATHNAME), 0);
#else
APSARA_TEST_EQUAL(
fnmatch((pathRoot + PS + "6" + PS + "*" + PS + "*" + PS + "1").c_str(), tmpPath.c_str(), FNM_PATHNAME), 1);
ADD_CASE(tmpPath, "");
#endif
// Won't block by rule 5 in directory blacklist.
tmpPath = pathRoot + PS + "10";
ADD_CASE(tmpPath, "");
// Won't block by rule 6 in directory blacklist.
tmpPath = pathRoot + PS + "11" + PS + "1";
ADD_CASE(tmpPath, "");
tmpPath = pathRoot + PS + "11" + PS + "2";
ADD_CASE(tmpPath, "");
tmpPath = pathRoot + PS + "11" + PS + "a" + PS + "2";
ADD_CASE(tmpPath, "");
tmpPath = pathRoot + PS + "11" + PS + "a" + PS + "b" + PS + "3";
ADD_CASE(tmpPath, "");
// Won't block by rule 3 in file path blacklist.
tmpPath = pathRoot + PS + "7";
ADD_CASE(tmpPath, "ignore.log");
#if !defined(_MSC_VER)
tmpPath = pathRoot + PS + "7" + PS + "4" + PS + "a";
ADD_CASE(tmpPath, "ignore.log");
#endif
// Won't block by rule 4 in file path blacklist.
tmpPath = pathRoot + PS + "8" + PS + "1";
ADD_CASE(tmpPath, "ignore.log");
#if !defined(_MSC_VER)
tmpPath = pathRoot + PS + "8" + PS + "a" + PS + "b" + PS + "1";
ADD_CASE(tmpPath, "ignore.log");
#endif
// Won't block by rule 5 in file path blacklist.
tmpPath = pathRoot + PS + "9" + PS + "1";
ADD_CASE(tmpPath, "ignore.log");
tmpPath = pathRoot + PS + "9" + PS + "a" + PS + "1";
ADD_CASE(tmpPath, "ignore.log");
#if !defined(_MSC_VER)
tmpPath = pathRoot + PS + "9" + PS + "a" + PS + "b" + PS + "c" + PS + "1";
ADD_CASE(tmpPath, "ignore.log");
#endif
// Won't block by rule 6 in file path blacklist.
tmpPath = pathRoot + PS + "12";
APSARA_TEST_EQUAL(fnmatch((pathRoot + PS + "12" + PS + "**" + PS + "ignore*.log").c_str(),
(tmpPath + PS + "ignore.log").c_str(),
0),
1);
ADD_CASE(tmpPath, "ignore.log");
// Won't block by rule 7 in file path blacklist.
tmpPath = pathRoot + PS + "13" + PS + "1";
APSARA_TEST_EQUAL(fnmatch((pathRoot + PS + "13" + PS + "**" + PS + "1" + PS + "ignore*.log").c_str(),
(tmpPath + PS + "ignore.log").c_str(),
0),
1);
ADD_CASE(tmpPath, "ignore.log");
tmpPath = pathRoot + PS + "13" + PS + "2";
ADD_CASE(tmpPath, "ignore.log");
tmpPath = pathRoot + PS + "13" + PS + "a" + PS + "2";
ADD_CASE(tmpPath, "ignore.log");
tmpPath = pathRoot + PS + "13" + PS + "a" + PS + "b" + PS + "3";
ADD_CASE(tmpPath, "ignore.log");
acceptedCases.push_back({pathRoot + PS + "2", "", true});
acceptedCases.push_back({pathRoot + PS + "3", "", true});
acceptedCases.push_back({pathRoot + PS + "3" + PS + "5", "", true});
acceptedCases.push_back({pathRoot + PS + "3" + PS + "5", "1.log", true});
acceptedCases.push_back({pathRoot + PS + "3" + PS + "5", "2.log", true});
acceptedCases.push_back({pathRoot + PS + "2" + PS + "2", "2.log", true});
acceptedCases.push_back({pathRoot + PS + "2" + PS + "3", "3.log", true});
acceptedCases.push_back({pathRoot + PS + "2" + PS + "3", "lignore.log", true});
for (size_t i = 0; i < acceptedCases.size(); ++i) {
Case& c = acceptedCases[i];
bool result = config.IsMatch(c.path, c.name);
LOG_INFO(sLogger, ("Accept case", i)("detail", c.String())("actual", result));
APSARA_TEST_EQUAL_DESC(c.expected, result, c.String());
}
#undef ADD_CASE
CaseCleanUp();
LOG_INFO(sLogger, ("TestBlacklistControlInIsMatch() end", time(NULL)));
}
APSARA_UNIT_TEST_CASE(ConfigMatchUnittest, TestBlacklistControlInIsMatch, 0);
// File not in docker and with normal base path.
void ConfigMatchUnittest::TestBlacklistControl() {
LOG_INFO(sLogger, ("TestBlacklistControl() begin", time(NULL)));
TestBlacklistControlCommon(gRootDir);
LOG_INFO(sLogger, ("TestBlacklistControl() end", time(NULL)));
}
APSARA_UNIT_TEST_CASE(ConfigMatchUnittest, TestBlacklistControl, 0);
// File not in docker but with wildcard path.
void ConfigMatchUnittest::TestBlacklistControlWildcardBasePath() {
LOG_INFO(sLogger, ("TestBlacklistControlWildcardBasePath() begin", time(NULL)));
auto& PS = PATH_SEPARATOR;
TestBlacklistControlCommon(gRootDir + PS + "test" + PS + "x", gRootDir + PS + "test*" + PS + "?");
LOG_INFO(sLogger, ("TestBlacklistControlWildcardBasePath() end", time(NULL)));
}
APSARA_UNIT_TEST_CASE(ConfigMatchUnittest, TestBlacklistControlWildcardBasePath, 0);
void ConfigMatchUnittest::TestChinesePathAndFilePattern() {
LOG_INFO(sLogger, ("TestChinesePathAndFilePattern() begin", time(NULL)));
CaseSetUp();
auto const pathRoot = gRootDir + PATH_SEPARATOR + "中文路径";
auto const fileNames
= std::vector<std::string>{"中文文件a.txt", "中文文件b中文.txt", "中文文件c.txt中", "中文文件d.txtt", "a.txt"};
auto const matchResults = std::vector<bool>{true, true, false, false, false};
bfs::create_directories(pathRoot);
for (auto& fn : fileNames) {
std::ofstream(pathRoot + PATH_SEPARATOR + fn) << "content";
}
std::string basePath = pathRoot;
std::string filePattern = "中文文件*.txt";
#if defined(_MSC_VER)
basePath = EncodingConverter::GetInstance()->FromACPToUTF8(basePath);
filePattern = EncodingConverter::GetInstance()->FromACPToUTF8(filePattern);
#endif
{
CollectionConfig cfg(
basePath, filePattern, LogType::REGEX_LOG, "log", ".*", "", "", "project", true, 3, 3, "logstore");
fsutil::Dir dir(pathRoot);
APSARA_TEST_TRUE(dir.Open());
while (fsutil::Entry ent = dir.ReadNext()) {
auto fn = ent.Name();
auto r = cfg.IsMatch(pathRoot, fn);
for (size_t i = 0; i < fileNames.size(); ++i) {
if (fileNames[i] == fn) {
APSARA_TEST_EQUAL_DESC(matchResults[i], r, fn);
}
}
}
}
CaseCleanUp();
LOG_INFO(sLogger, ("TestChinesePathAndFilePattern() end", time(NULL)));
}
APSARA_UNIT_TEST_CASE(ConfigMatchUnittest, TestChinesePathAndFilePattern, 0);
} // end of namespace logtail
int main(int argc, char** argv) {
logtail::Logger::Instance().InitGlobalLoggers();
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}