core/unittest/config/PipelineConfigWatcherUnittest.cpp (1,375 lines of code) (raw):
// Copyright 2024 iLogtail Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <memory>
#include <string>
#include "json/json.h"
#include "common/JsonUtil.h"
#include "config/watcher/PipelineConfigWatcher.h"
#include "plugin/PluginRegistry.h"
#include "unittest/Unittest.h"
#include "unittest/config/PipelineManagerMock.h"
#ifdef __ENTERPRISE__
#include "config/provider/EnterpriseConfigProvider.h"
#endif
using namespace std;
DECLARE_FLAG_BOOL(enable_ebpf_network_secure);
DECLARE_FLAG_BOOL(enable_ebpf_file_secure);
DECLARE_FLAG_BOOL(enable_ebpf_process_secure);
DECLARE_FLAG_BOOL(enable_ebpf_network_observer);
namespace logtail {
class PipelineConfigWatcherUnittest : public testing::Test {
public:
void TestLoadAddedSingletonConfig();
void TestLoadModifiedSingletonConfig();
void TestLoadRemovedSingletonConfig();
void TestLoadUnchangedSingletonConfig();
protected:
static void SetUpTestCase() {
FLAGS_enable_ebpf_network_observer = true;
FLAGS_enable_ebpf_process_secure = true;
FLAGS_enable_ebpf_file_secure = true;
FLAGS_enable_ebpf_network_secure = true;
PluginRegistry::GetInstance()->LoadPlugins();
PipelineConfigWatcher::GetInstance()->SetPipelineManager(PipelineManagerMock::GetInstance());
}
static void TearDownTestCase() { PluginRegistry::GetInstance()->UnloadPlugins(); }
private:
void PrepareConfig() {
filesystem::create_directories(configDir1);
PipelineConfigWatcher::GetInstance()->AddSource(configDir1.string());
filesystem::create_directories(configDir2);
PipelineConfigWatcher::GetInstance()->AddSource(configDir2.string());
#ifdef __ENTERPRISE__
builtinPipelineCnt = EnterpriseConfigProvider::GetInstance()->GetAllBuiltInPipelineConfigs().size();
#endif
}
void ClearConfig() {
PipelineManagerMock::GetInstance()->ClearEnvironment();
PipelineConfigWatcher::GetInstance()->ClearEnvironment();
filesystem::remove_all(configDir1);
filesystem::remove_all(configDir2);
}
size_t builtinPipelineCnt = 0;
filesystem::path configDir1 = "./continuous_pipeline_config1";
filesystem::path configDir2 = "./continuous_pipeline_config2";
const std::string greaterPriorityConfig = R"(
{
"createTime": 1,
"valid": true,
"inputs": [
{
"Type": "input_network_observer"
}
],
"flushers": [
{
"Type": "flusher_sls"
}
]
}
)";
const std::string lessPriorityConfig = R"(
{
"createTime": 2,
"valid": true,
"inputs": [
{
"Type": "input_network_observer"
}
],
"flushers": [
{
"Type": "flusher_sls"
}
]
}
)";
const std::string modifiedGreaterPriorityConfig = R"(
{
"createTime": 1,
"valid": true,
"inputs": [
{
"Type": "input_network_observer"
}
],
"processors": [],
"flushers": [
{
"Type": "flusher_sls"
}
]
}
)";
const std::string modifiedLessPriorityConfig = R"(
{
"createTime": 2,
"valid": true,
"inputs": [
{
"Type": "input_network_observer"
}
],
"processors": [],
"flushers": [
{
"Type": "flusher_sls"
}
]
}
)";
const std::string otherConfig = R"(
{
"createTime": 3,
"valid": true,
"inputs": [
{
"Type": "input_process_security"
}
],
"flushers": [
{
"Type": "flusher_sls"
}
]
}
)";
const std::string modifiedOtherConfig = R"(
{
"createTime": 3,
"valid": true,
"inputs": [
{
"Type": "input_process_security"
}
],
"processors": [],
"flushers": [
{
"Type": "flusher_sls"
}
]
}
)";
};
// there are 4 kinds of a config: added, modified, removed, unchanged
// there are 4 kinds of priority relationship: first > second, first < second,
// first > second -> first < second, first < second -> first > second
// total case: 4 (first kind) * 4(second kind) * 4(priority) = 64
void PipelineConfigWatcherUnittest::TestLoadAddedSingletonConfig() {
{ // case: added -> added, first > second
PrepareConfig();
ofstream fout(configDir1 / "test1.json", ios::trunc);
fout << greaterPriorityConfig;
fout.close();
fout.open(configDir2 / "test2.json", ios::trunc);
fout << lessPriorityConfig;
fout.close();
fout.open(configDir2 / "test-other.json", ios::trunc);
fout << otherConfig;
fout.close();
auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff();
APSARA_TEST_EQUAL_FATAL(2U + builtinPipelineCnt, diff.first.mAdded.size());
APSARA_TEST_EQUAL_FATAL(0U, diff.first.mModified.size());
APSARA_TEST_EQUAL_FATAL(0U, diff.first.mRemoved.size());
PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first);
auto allConfigNames = PipelineManagerMock::GetInstance()->GetAllConfigNames();
APSARA_TEST_EQUAL_FATAL(2U + builtinPipelineCnt, allConfigNames.size());
sort(allConfigNames.begin(), allConfigNames.end());
APSARA_TEST_EQUAL_FATAL("test-other", allConfigNames[builtinPipelineCnt]);
APSARA_TEST_EQUAL_FATAL("test1", allConfigNames[builtinPipelineCnt + 1]);
ClearConfig();
}
{ // case: added -> added, first < second
PrepareConfig();
ofstream fout(configDir1 / "test1.json", ios::trunc);
fout << lessPriorityConfig;
fout.close();
fout.open(configDir2 / "test2.json", ios::trunc);
fout << greaterPriorityConfig;
fout.close();
fout.open(configDir2 / "test-other.json", ios::trunc);
fout << otherConfig;
fout.close();
auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff();
APSARA_TEST_EQUAL_FATAL(2U + builtinPipelineCnt, diff.first.mAdded.size());
APSARA_TEST_EQUAL_FATAL(0U, diff.first.mModified.size());
APSARA_TEST_EQUAL_FATAL(0U, diff.first.mRemoved.size());
PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first);
auto allConfigNames = PipelineManagerMock::GetInstance()->GetAllConfigNames();
APSARA_TEST_EQUAL_FATAL(2U + builtinPipelineCnt, allConfigNames.size());
sort(allConfigNames.begin(), allConfigNames.end());
APSARA_TEST_EQUAL_FATAL("test-other", allConfigNames[builtinPipelineCnt]);
APSARA_TEST_EQUAL_FATAL("test2", allConfigNames[builtinPipelineCnt + 1]);
ClearConfig();
}
{
// case: added -> added, first > second -> first < second
// should not happen
}
{
// case: added -> added, first < second -> first > second
// should not happen
}
{ // case: added -> modified, first > second
PrepareConfig();
ofstream fout(configDir2 / "test2.json", ios::trunc);
fout << lessPriorityConfig;
fout.close();
fout.open(configDir2 / "test-other.json", ios::trunc);
fout << otherConfig;
fout.close();
auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff();
PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first);
APSARA_TEST_EQUAL_FATAL(2U + builtinPipelineCnt,
PipelineManagerMock::GetInstance()->GetAllConfigNames().size());
std::this_thread::sleep_for(std::chrono::milliseconds(1));
fout.open(configDir1 / "test1.json", ios::trunc);
fout << greaterPriorityConfig;
fout.close();
fout.open(configDir2 / "test2.json", ios::trunc);
fout << modifiedLessPriorityConfig;
fout.close();
fout.open(configDir2 / "test-other.json", ios::trunc);
fout << modifiedOtherConfig;
fout.close();
diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff();
APSARA_TEST_EQUAL_FATAL(1U, diff.first.mAdded.size());
APSARA_TEST_EQUAL_FATAL(1U, diff.first.mModified.size());
APSARA_TEST_EQUAL_FATAL(1U, diff.first.mRemoved.size());
PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first);
auto allConfigNames = PipelineManagerMock::GetInstance()->GetAllConfigNames();
APSARA_TEST_EQUAL_FATAL(2U + builtinPipelineCnt, allConfigNames.size());
sort(allConfigNames.begin(), allConfigNames.end());
APSARA_TEST_EQUAL_FATAL("test-other", allConfigNames[builtinPipelineCnt]);
APSARA_TEST_EQUAL_FATAL("test1", allConfigNames[builtinPipelineCnt + 1]);
ClearConfig();
}
{ // case: added -> modified, first < second
PrepareConfig();
ofstream fout(configDir2 / "test2.json", ios::trunc);
fout << greaterPriorityConfig;
fout.close();
fout.open(configDir2 / "test-other.json", ios::trunc);
fout << otherConfig;
fout.close();
auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff();
PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first);
APSARA_TEST_EQUAL_FATAL(2U + builtinPipelineCnt,
PipelineManagerMock::GetInstance()->GetAllConfigNames().size());
std::this_thread::sleep_for(std::chrono::milliseconds(1));
fout.open(configDir1 / "test1.json", ios::trunc);
fout << lessPriorityConfig;
fout.close();
fout.open(configDir2 / "test2.json", ios::trunc);
fout << modifiedGreaterPriorityConfig;
fout.close();
fout.open(configDir2 / "test-other.json", ios::trunc);
fout << modifiedOtherConfig;
fout.close();
diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff();
APSARA_TEST_EQUAL_FATAL(0U, diff.first.mAdded.size());
APSARA_TEST_EQUAL_FATAL(2U, diff.first.mModified.size());
APSARA_TEST_EQUAL_FATAL(0U, diff.first.mRemoved.size());
PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first);
auto allConfigNames = PipelineManagerMock::GetInstance()->GetAllConfigNames();
APSARA_TEST_EQUAL_FATAL(2U + builtinPipelineCnt, allConfigNames.size());
sort(allConfigNames.begin(), allConfigNames.end());
APSARA_TEST_EQUAL_FATAL("test-other", allConfigNames[builtinPipelineCnt]);
APSARA_TEST_EQUAL_FATAL("test2", allConfigNames[builtinPipelineCnt + 1]);
ClearConfig();
}
{
// case: added -> modified, first > second -> first < second
// should not happen
}
{
// case: added -> modified, first < second -> first > second
// should not happen
}
{ // case: added -> removed, first > second
PrepareConfig();
ofstream fout(configDir2 / "test2.json", ios::trunc);
fout << lessPriorityConfig;
fout.close();
fout.open(configDir2 / "test-other.json", ios::trunc);
fout << otherConfig;
fout.close();
auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff();
PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first);
APSARA_TEST_EQUAL_FATAL(2U + builtinPipelineCnt,
PipelineManagerMock::GetInstance()->GetAllConfigNames().size());
fout.open(configDir1 / "test1.json", ios::trunc);
fout << greaterPriorityConfig;
fout.close();
filesystem::remove(configDir2 / "test2.json");
filesystem::remove(configDir2 / "test-other.json");
diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff();
APSARA_TEST_EQUAL_FATAL(1U, diff.first.mAdded.size());
APSARA_TEST_EQUAL_FATAL(0U, diff.first.mModified.size());
APSARA_TEST_EQUAL_FATAL(2U, diff.first.mRemoved.size());
PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first);
auto allConfigNames = PipelineManagerMock::GetInstance()->GetAllConfigNames();
APSARA_TEST_EQUAL_FATAL(1U + builtinPipelineCnt, allConfigNames.size());
APSARA_TEST_EQUAL_FATAL("test1", allConfigNames[0]);
ClearConfig();
}
{ // case: added -> removed, first < second
PrepareConfig();
ofstream fout(configDir2 / "test2.json", ios::trunc);
fout << greaterPriorityConfig;
fout.close();
fout.open(configDir2 / "test-other.json", ios::trunc);
fout << otherConfig;
fout.close();
auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff();
PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first);
APSARA_TEST_EQUAL_FATAL(2U + builtinPipelineCnt,
PipelineManagerMock::GetInstance()->GetAllConfigNames().size());
fout.open(configDir1 / "test1.json", ios::trunc);
fout << lessPriorityConfig;
fout.close();
filesystem::remove(configDir2 / "test2.json");
filesystem::remove(configDir2 / "test-other.json");
diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff();
APSARA_TEST_EQUAL_FATAL(1U, diff.first.mAdded.size());
APSARA_TEST_EQUAL_FATAL(0U, diff.first.mModified.size());
APSARA_TEST_EQUAL_FATAL(2U, diff.first.mRemoved.size());
PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first);
auto allConfigNames = PipelineManagerMock::GetInstance()->GetAllConfigNames();
APSARA_TEST_EQUAL_FATAL(1U + builtinPipelineCnt, allConfigNames.size());
APSARA_TEST_EQUAL_FATAL("test1", allConfigNames[0]);
ClearConfig();
}
{
// case: added -> removed, first > second -> first < second
// should not happen
}
{
// case: added -> removed, first < second -> first > second
// should not happen
}
{ // case: added -> unchanged, first > second
PrepareConfig();
ofstream fout(configDir2 / "test2.json", ios::trunc);
fout << lessPriorityConfig;
fout.close();
fout.open(configDir2 / "test-other.json", ios::trunc);
fout << otherConfig;
fout.close();
auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff();
PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first);
APSARA_TEST_EQUAL_FATAL(2U + builtinPipelineCnt,
PipelineManagerMock::GetInstance()->GetAllConfigNames().size());
fout.open(configDir1 / "test1.json", ios::trunc);
fout << greaterPriorityConfig;
fout.close();
diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff();
APSARA_TEST_EQUAL_FATAL(1U, diff.first.mAdded.size());
APSARA_TEST_EQUAL_FATAL(0U, diff.first.mModified.size());
APSARA_TEST_EQUAL_FATAL(1U, diff.first.mRemoved.size());
PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first);
auto allConfigNames = PipelineManagerMock::GetInstance()->GetAllConfigNames();
APSARA_TEST_EQUAL_FATAL(2U + builtinPipelineCnt, allConfigNames.size());
sort(allConfigNames.begin(), allConfigNames.end());
APSARA_TEST_EQUAL_FATAL("test-other", allConfigNames[builtinPipelineCnt]);
APSARA_TEST_EQUAL_FATAL("test1", allConfigNames[builtinPipelineCnt + 1]);
ClearConfig();
}
{ // case: added -> unchanged, first < second
PrepareConfig();
ofstream fout(configDir2 / "test2.json", ios::trunc);
fout << greaterPriorityConfig;
fout.close();
fout.open(configDir2 / "test-other.json", ios::trunc);
fout << otherConfig;
fout.close();
auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff();
PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first);
APSARA_TEST_EQUAL_FATAL(2U + builtinPipelineCnt,
PipelineManagerMock::GetInstance()->GetAllConfigNames().size());
fout.open(configDir1 / "test1.json", ios::trunc);
fout << lessPriorityConfig;
fout.close();
diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff();
APSARA_TEST_EQUAL_FATAL(0U, diff.first.mAdded.size());
APSARA_TEST_EQUAL_FATAL(0U, diff.first.mModified.size());
APSARA_TEST_EQUAL_FATAL(0U, diff.first.mRemoved.size());
PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first);
auto allConfigNames = PipelineManagerMock::GetInstance()->GetAllConfigNames();
APSARA_TEST_EQUAL_FATAL(2U + builtinPipelineCnt, allConfigNames.size());
sort(allConfigNames.begin(), allConfigNames.end());
APSARA_TEST_EQUAL_FATAL("test-other", allConfigNames[builtinPipelineCnt]);
APSARA_TEST_EQUAL_FATAL("test2", allConfigNames[builtinPipelineCnt + 1]);
ClearConfig();
}
{
// case: added -> unchanged, first > second -> first < second
// should not happen
}
{ // case: added -> unchanged, first < second -> first > second
// should not happen
}
}
void PipelineConfigWatcherUnittest::TestLoadModifiedSingletonConfig() {
{ // case: modified -> added, first > second
PrepareConfig();
ofstream fout(configDir1 / "test1.json", ios::trunc);
fout << greaterPriorityConfig;
fout.close();
auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff();
PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first);
APSARA_TEST_EQUAL_FATAL(1U + builtinPipelineCnt,
PipelineManagerMock::GetInstance()->GetAllConfigNames().size());
std::this_thread::sleep_for(std::chrono::milliseconds(1));
fout.open(configDir1 / "test1.json", ios::trunc);
fout << modifiedGreaterPriorityConfig;
fout.close();
fout.open(configDir2 / "test2.json", ios::trunc);
fout << lessPriorityConfig;
fout.close();
fout.open(configDir2 / "test-other.json", ios::trunc);
fout << otherConfig;
fout.close();
diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff();
APSARA_TEST_EQUAL_FATAL(1U, diff.first.mAdded.size());
APSARA_TEST_EQUAL_FATAL(1U, diff.first.mModified.size());
APSARA_TEST_EQUAL_FATAL(0U, diff.first.mRemoved.size());
PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first);
auto allConfigNames = PipelineManagerMock::GetInstance()->GetAllConfigNames();
APSARA_TEST_EQUAL_FATAL(2U + builtinPipelineCnt, allConfigNames.size());
sort(allConfigNames.begin(), allConfigNames.end());
APSARA_TEST_EQUAL_FATAL("test-other", allConfigNames[builtinPipelineCnt]);
APSARA_TEST_EQUAL_FATAL("test1", allConfigNames[builtinPipelineCnt + 1]);
ClearConfig();
}
{ // case: modified -> added, first < second
PrepareConfig();
ofstream fout(configDir1 / "test1.json", ios::trunc);
fout << lessPriorityConfig;
fout.close();
auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff();
PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first);
APSARA_TEST_EQUAL_FATAL(1U + builtinPipelineCnt,
PipelineManagerMock::GetInstance()->GetAllConfigNames().size());
std::this_thread::sleep_for(std::chrono::milliseconds(1));
fout.open(configDir1 / "test1.json", ios::trunc);
fout << modifiedLessPriorityConfig;
fout.close();
fout.open(configDir2 / "test2.json", ios::trunc);
fout << greaterPriorityConfig;
fout.close();
fout.open(configDir2 / "test-other.json", ios::trunc);
fout << otherConfig;
fout.close();
diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff();
APSARA_TEST_EQUAL_FATAL(2U, diff.first.mAdded.size());
APSARA_TEST_EQUAL_FATAL(0U, diff.first.mModified.size());
APSARA_TEST_EQUAL_FATAL(1U, diff.first.mRemoved.size());
PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first);
auto allConfigNames = PipelineManagerMock::GetInstance()->GetAllConfigNames();
APSARA_TEST_EQUAL_FATAL(2U + builtinPipelineCnt, allConfigNames.size());
sort(allConfigNames.begin(), allConfigNames.end());
APSARA_TEST_EQUAL_FATAL("test-other", allConfigNames[builtinPipelineCnt]);
APSARA_TEST_EQUAL_FATAL("test2", allConfigNames[builtinPipelineCnt + 1]);
ClearConfig();
}
{
// case: modified -> added, first > second -> first < second
// should not happen
}
{
// case: modified -> added, first < second -> first > second
// should not happen
}
{ // case: modified -> modified, first > second
PrepareConfig();
ofstream fout(configDir1 / "test1.json", ios::trunc);
fout << greaterPriorityConfig;
fout.close();
fout.open(configDir2 / "test2.json", ios::trunc);
fout << lessPriorityConfig;
fout.close();
fout.open(configDir2 / "test-other.json", ios::trunc);
fout << otherConfig;
fout.close();
auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff();
PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first);
APSARA_TEST_EQUAL_FATAL(2U + builtinPipelineCnt,
PipelineManagerMock::GetInstance()->GetAllConfigNames().size());
std::this_thread::sleep_for(std::chrono::milliseconds(1));
fout.open(configDir1 / "test1.json", ios::trunc);
fout << modifiedGreaterPriorityConfig;
fout.close();
fout.open(configDir2 / "test2.json", ios::trunc);
fout << modifiedLessPriorityConfig;
fout.close();
fout.open(configDir2 / "test-other.json", ios::trunc);
fout << modifiedOtherConfig;
fout.close();
diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff();
APSARA_TEST_EQUAL_FATAL(0U, diff.first.mAdded.size());
APSARA_TEST_EQUAL_FATAL(2U, diff.first.mModified.size());
APSARA_TEST_EQUAL_FATAL(0U, diff.first.mRemoved.size());
PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first);
auto allConfigNames = PipelineManagerMock::GetInstance()->GetAllConfigNames();
APSARA_TEST_EQUAL_FATAL(2U + builtinPipelineCnt, allConfigNames.size());
sort(allConfigNames.begin(), allConfigNames.end());
APSARA_TEST_EQUAL_FATAL("test-other", allConfigNames[builtinPipelineCnt]);
APSARA_TEST_EQUAL_FATAL("test1", allConfigNames[builtinPipelineCnt + 1]);
ClearConfig();
}
{ // case: modified -> modified, first < second
PrepareConfig();
ofstream fout(configDir1 / "test1.json", ios::trunc);
fout << lessPriorityConfig;
fout.close();
fout.open(configDir2 / "test2.json", ios::trunc);
fout << greaterPriorityConfig;
fout.close();
fout.open(configDir2 / "test-other.json", ios::trunc);
fout << otherConfig;
fout.close();
auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff();
PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first);
APSARA_TEST_EQUAL_FATAL(2U + builtinPipelineCnt,
PipelineManagerMock::GetInstance()->GetAllConfigNames().size());
std::this_thread::sleep_for(std::chrono::milliseconds(1));
fout.open(configDir1 / "test1.json", ios::trunc);
fout << modifiedLessPriorityConfig;
fout.close();
fout.open(configDir2 / "test2.json", ios::trunc);
fout << modifiedGreaterPriorityConfig;
fout.close();
fout.open(configDir2 / "test-other.json", ios::trunc);
fout << modifiedOtherConfig;
fout.close();
diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff();
APSARA_TEST_EQUAL_FATAL(0U, diff.first.mAdded.size());
APSARA_TEST_EQUAL_FATAL(2U, diff.first.mModified.size());
APSARA_TEST_EQUAL_FATAL(0U, diff.first.mRemoved.size());
PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first);
auto allConfigNames = PipelineManagerMock::GetInstance()->GetAllConfigNames();
APSARA_TEST_EQUAL_FATAL(2U + builtinPipelineCnt, allConfigNames.size());
sort(allConfigNames.begin(), allConfigNames.end());
APSARA_TEST_EQUAL_FATAL("test-other", allConfigNames[builtinPipelineCnt]);
APSARA_TEST_EQUAL_FATAL("test2", allConfigNames[builtinPipelineCnt + 1]);
ClearConfig();
}
{ // case: modified -> modified, first > second -> first < second
PrepareConfig();
ofstream fout(configDir1 / "test1.json", ios::trunc);
fout << greaterPriorityConfig;
fout.close();
fout.open(configDir2 / "test2.json", ios::trunc);
fout << lessPriorityConfig;
fout.close();
fout.open(configDir2 / "test-other.json", ios::trunc);
fout << otherConfig;
fout.close();
auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff();
PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first);
APSARA_TEST_EQUAL_FATAL(2U + builtinPipelineCnt,
PipelineManagerMock::GetInstance()->GetAllConfigNames().size());
std::this_thread::sleep_for(std::chrono::milliseconds(1));
fout.open(configDir1 / "test1.json", ios::trunc);
fout << modifiedLessPriorityConfig;
fout.close();
fout.open(configDir2 / "test2.json", ios::trunc);
fout << modifiedGreaterPriorityConfig;
fout.close();
fout.open(configDir2 / "test-other.json", ios::trunc);
fout << modifiedOtherConfig;
fout.close();
diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff();
APSARA_TEST_EQUAL_FATAL(1U, diff.first.mAdded.size());
APSARA_TEST_EQUAL_FATAL(1U, diff.first.mModified.size());
APSARA_TEST_EQUAL_FATAL(1U, diff.first.mRemoved.size());
PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first);
auto allConfigNames = PipelineManagerMock::GetInstance()->GetAllConfigNames();
APSARA_TEST_EQUAL_FATAL(2U + builtinPipelineCnt, allConfigNames.size());
sort(allConfigNames.begin(), allConfigNames.end());
APSARA_TEST_EQUAL_FATAL("test-other", allConfigNames[builtinPipelineCnt]);
APSARA_TEST_EQUAL_FATAL("test2", allConfigNames[builtinPipelineCnt + 1]);
ClearConfig();
}
{ // case: modified -> modified, first < second -> first > second
PrepareConfig();
ofstream fout(configDir1 / "test1.json", ios::trunc);
fout << lessPriorityConfig;
fout.close();
fout.open(configDir2 / "test2.json", ios::trunc);
fout << greaterPriorityConfig;
fout.close();
fout.open(configDir2 / "test-other.json", ios::trunc);
fout << otherConfig;
fout.close();
auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff();
PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first);
APSARA_TEST_EQUAL_FATAL(2U + builtinPipelineCnt,
PipelineManagerMock::GetInstance()->GetAllConfigNames().size());
std::this_thread::sleep_for(std::chrono::milliseconds(1));
fout.open(configDir1 / "test1.json", ios::trunc);
fout << modifiedGreaterPriorityConfig;
fout.close();
fout.open(configDir2 / "test2.json", ios::trunc);
fout << modifiedLessPriorityConfig;
fout.close();
fout.open(configDir2 / "test-other.json", ios::trunc);
fout << modifiedOtherConfig;
fout.close();
diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff();
APSARA_TEST_EQUAL_FATAL(1U, diff.first.mAdded.size());
APSARA_TEST_EQUAL_FATAL(1U, diff.first.mModified.size());
APSARA_TEST_EQUAL_FATAL(1U, diff.first.mRemoved.size());
PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first);
auto allConfigNames = PipelineManagerMock::GetInstance()->GetAllConfigNames();
APSARA_TEST_EQUAL_FATAL(2U + builtinPipelineCnt, allConfigNames.size());
sort(allConfigNames.begin(), allConfigNames.end());
APSARA_TEST_EQUAL_FATAL("test-other", allConfigNames[builtinPipelineCnt]);
APSARA_TEST_EQUAL_FATAL("test1", allConfigNames[builtinPipelineCnt + 1]);
ClearConfig();
}
{ // case: modified -> removed, first > second
PrepareConfig();
ofstream fout(configDir1 / "test1.json", ios::trunc);
fout << greaterPriorityConfig;
fout.close();
fout.open(configDir2 / "test2.json", ios::trunc);
fout << lessPriorityConfig;
fout.close();
fout.open(configDir2 / "test-other.json", ios::trunc);
fout << otherConfig;
fout.close();
auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff();
PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first);
APSARA_TEST_EQUAL_FATAL(2U + builtinPipelineCnt,
PipelineManagerMock::GetInstance()->GetAllConfigNames().size());
std::this_thread::sleep_for(std::chrono::milliseconds(1));
fout.open(configDir1 / "test1.json", ios::trunc);
fout << modifiedGreaterPriorityConfig;
fout.close();
filesystem::remove(configDir2 / "test2.json");
filesystem::remove(configDir2 / "test-other.json");
diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff();
APSARA_TEST_EQUAL_FATAL(0U, diff.first.mAdded.size());
APSARA_TEST_EQUAL_FATAL(1U, diff.first.mModified.size());
APSARA_TEST_EQUAL_FATAL(1U, diff.first.mRemoved.size());
PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first);
auto allConfigNames = PipelineManagerMock::GetInstance()->GetAllConfigNames();
APSARA_TEST_EQUAL_FATAL(1U + builtinPipelineCnt, allConfigNames.size());
APSARA_TEST_EQUAL_FATAL("test1", allConfigNames[builtinPipelineCnt]);
ClearConfig();
}
{ // case: modified -> removed, first < second
PrepareConfig();
ofstream fout(configDir1 / "test1.json", ios::trunc);
fout << lessPriorityConfig;
fout.close();
fout.open(configDir2 / "test2.json", ios::trunc);
fout << greaterPriorityConfig;
fout.close();
fout.open(configDir2 / "test-other.json", ios::trunc);
fout << otherConfig;
fout.close();
auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff();
PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first);
APSARA_TEST_EQUAL_FATAL(2U + builtinPipelineCnt,
PipelineManagerMock::GetInstance()->GetAllConfigNames().size());
std::this_thread::sleep_for(std::chrono::milliseconds(1));
fout.open(configDir1 / "test1.json", ios::trunc);
fout << modifiedLessPriorityConfig;
fout.close();
filesystem::remove(configDir2 / "test2.json");
filesystem::remove(configDir2 / "test-other.json");
diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff();
APSARA_TEST_EQUAL_FATAL(1U, diff.first.mAdded.size());
APSARA_TEST_EQUAL_FATAL(0U, diff.first.mModified.size());
APSARA_TEST_EQUAL_FATAL(2U, diff.first.mRemoved.size());
PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first);
auto allConfigNames = PipelineManagerMock::GetInstance()->GetAllConfigNames();
APSARA_TEST_EQUAL_FATAL(1U + builtinPipelineCnt, allConfigNames.size());
APSARA_TEST_EQUAL_FATAL("test1", allConfigNames[0]);
ClearConfig();
}
{
// case: modified -> removed, first > second -> first < second
// should not happen
}
{
// case: modified -> removed, first < second -> first > second
// should not happen
}
{ // case: modified -> unchanged, first > second
PrepareConfig();
ofstream fout(configDir1 / "test1.json", ios::trunc);
fout << greaterPriorityConfig;
fout.close();
fout.open(configDir2 / "test2.json", ios::trunc);
fout << lessPriorityConfig;
fout.close();
fout.open(configDir2 / "test-other.json", ios::trunc);
fout << otherConfig;
fout.close();
auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff();
PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first);
APSARA_TEST_EQUAL_FATAL(2U + builtinPipelineCnt,
PipelineManagerMock::GetInstance()->GetAllConfigNames().size());
std::this_thread::sleep_for(std::chrono::milliseconds(1));
fout.open(configDir1 / "test1.json", ios::trunc);
fout << modifiedGreaterPriorityConfig;
fout.close();
diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff();
APSARA_TEST_EQUAL_FATAL(0U, diff.first.mAdded.size());
APSARA_TEST_EQUAL_FATAL(1U, diff.first.mModified.size());
APSARA_TEST_EQUAL_FATAL(0U, diff.first.mRemoved.size());
PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first);
auto allConfigNames = PipelineManagerMock::GetInstance()->GetAllConfigNames();
APSARA_TEST_EQUAL_FATAL(2U + builtinPipelineCnt, allConfigNames.size());
sort(allConfigNames.begin(), allConfigNames.end());
APSARA_TEST_EQUAL_FATAL("test-other", allConfigNames[builtinPipelineCnt]);
APSARA_TEST_EQUAL_FATAL("test1", allConfigNames[builtinPipelineCnt + 1]);
ClearConfig();
}
{ // case: modified -> unchanged, first < second
PrepareConfig();
ofstream fout(configDir1 / "test1.json", ios::trunc);
fout << lessPriorityConfig;
fout.close();
fout.open(configDir2 / "test2.json", ios::trunc);
fout << greaterPriorityConfig;
fout.close();
fout.open(configDir2 / "test-other.json", ios::trunc);
fout << otherConfig;
fout.close();
auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff();
PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first);
APSARA_TEST_EQUAL_FATAL(2U + builtinPipelineCnt,
PipelineManagerMock::GetInstance()->GetAllConfigNames().size());
std::this_thread::sleep_for(std::chrono::milliseconds(1));
fout.open(configDir1 / "test1.json", ios::trunc);
fout << modifiedLessPriorityConfig;
fout.close();
filesystem::remove(configDir2 / "test2.json");
diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff();
APSARA_TEST_EQUAL_FATAL(1U, diff.first.mAdded.size());
APSARA_TEST_EQUAL_FATAL(0U, diff.first.mModified.size());
APSARA_TEST_EQUAL_FATAL(1U, diff.first.mRemoved.size());
PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first);
auto allConfigNames = PipelineManagerMock::GetInstance()->GetAllConfigNames();
APSARA_TEST_EQUAL_FATAL(2U + builtinPipelineCnt, allConfigNames.size());
sort(allConfigNames.begin(), allConfigNames.end());
APSARA_TEST_EQUAL_FATAL("test-other", allConfigNames[builtinPipelineCnt]);
APSARA_TEST_EQUAL_FATAL("test1", allConfigNames[builtinPipelineCnt + 1]);
ClearConfig();
}
{ // case: modified -> unchanged, first > second -> first < second
PrepareConfig();
ofstream fout(configDir1 / "test1.json", ios::trunc);
fout << greaterPriorityConfig;
fout.close();
fout.open(configDir2 / "test2.json", ios::trunc);
fout << greaterPriorityConfig;
fout.close();
fout.open(configDir2 / "test-other.json", ios::trunc);
fout << otherConfig;
fout.close();
auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff();
PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first);
APSARA_TEST_EQUAL_FATAL(2U + builtinPipelineCnt,
PipelineManagerMock::GetInstance()->GetAllConfigNames().size());
std::this_thread::sleep_for(std::chrono::milliseconds(1));
fout.open(configDir1 / "test1.json", ios::trunc);
fout << lessPriorityConfig;
fout.close();
diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff();
APSARA_TEST_EQUAL_FATAL(1U, diff.first.mAdded.size());
APSARA_TEST_EQUAL_FATAL(0U, diff.first.mModified.size());
APSARA_TEST_EQUAL_FATAL(1U, diff.first.mRemoved.size());
PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first);
auto allConfigNames = PipelineManagerMock::GetInstance()->GetAllConfigNames();
APSARA_TEST_EQUAL_FATAL(2U + builtinPipelineCnt, allConfigNames.size());
sort(allConfigNames.begin(), allConfigNames.end());
APSARA_TEST_EQUAL_FATAL("test-other", allConfigNames[builtinPipelineCnt]);
APSARA_TEST_EQUAL_FATAL("test2", allConfigNames[builtinPipelineCnt + 1]);
ClearConfig();
}
{ // case: modified -> unchanged, first < second -> first > second
PrepareConfig();
ofstream fout(configDir1 / "test3.json", ios::trunc);
fout << lessPriorityConfig;
fout.close();
fout.open(configDir2 / "test2.json", ios::trunc);
fout << lessPriorityConfig;
fout.close();
fout.open(configDir2 / "test-other.json", ios::trunc);
fout << otherConfig;
fout.close();
auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff();
PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first);
APSARA_TEST_EQUAL_FATAL(2U + builtinPipelineCnt,
PipelineManagerMock::GetInstance()->GetAllConfigNames().size());
std::this_thread::sleep_for(std::chrono::milliseconds(1));
fout.open(configDir1 / "test3.json", ios::trunc);
fout << greaterPriorityConfig;
fout.close();
diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff();
APSARA_TEST_EQUAL_FATAL(1U, diff.first.mAdded.size());
APSARA_TEST_EQUAL_FATAL(0U, diff.first.mModified.size());
APSARA_TEST_EQUAL_FATAL(1U, diff.first.mRemoved.size());
PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first);
auto allConfigNames = PipelineManagerMock::GetInstance()->GetAllConfigNames();
APSARA_TEST_EQUAL_FATAL(2U + builtinPipelineCnt, allConfigNames.size());
sort(allConfigNames.begin(), allConfigNames.end());
APSARA_TEST_EQUAL_FATAL("test-other", allConfigNames[builtinPipelineCnt]);
APSARA_TEST_EQUAL_FATAL("test3", allConfigNames[builtinPipelineCnt + 1]);
ClearConfig();
}
}
void PipelineConfigWatcherUnittest::TestLoadRemovedSingletonConfig() {
{ // case: removed -> added, first > second
PrepareConfig();
ofstream fout(configDir1 / "test1.json", ios::trunc);
fout << greaterPriorityConfig;
fout.close();
auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff();
size_t builtinPipelineCnt = 0;
#ifdef __ENTERPRISE__
builtinPipelineCnt += EnterpriseConfigProvider::GetInstance()->GetAllBuiltInPipelineConfigs().size();
#endif
PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first);
APSARA_TEST_EQUAL_FATAL(1U + builtinPipelineCnt,
PipelineManagerMock::GetInstance()->GetAllConfigNames().size());
filesystem::remove(configDir1 / "test1.json");
fout.open(configDir2 / "test2.json", ios::trunc);
fout << lessPriorityConfig;
fout.close();
fout.open(configDir2 / "test-other.json", ios::trunc);
fout << otherConfig;
fout.close();
diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff();
APSARA_TEST_EQUAL_FATAL(2U, diff.first.mAdded.size());
APSARA_TEST_EQUAL_FATAL(0U, diff.first.mModified.size());
APSARA_TEST_EQUAL_FATAL(1U, diff.first.mRemoved.size());
PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first);
auto allConfigNames = PipelineManagerMock::GetInstance()->GetAllConfigNames();
APSARA_TEST_EQUAL_FATAL(2U + builtinPipelineCnt, allConfigNames.size());
sort(allConfigNames.begin(), allConfigNames.end());
APSARA_TEST_EQUAL_FATAL("test-other", allConfigNames[builtinPipelineCnt]);
APSARA_TEST_EQUAL_FATAL("test2", allConfigNames[builtinPipelineCnt + 1]);
ClearConfig();
}
{ // case: removed -> added, first < second
PrepareConfig();
ofstream fout(configDir1 / "test1.json", ios::trunc);
fout << lessPriorityConfig;
fout.close();
auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff();
PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first);
APSARA_TEST_EQUAL_FATAL(1U + builtinPipelineCnt,
PipelineManagerMock::GetInstance()->GetAllConfigNames().size());
filesystem::remove(configDir1 / "test1.json");
fout.open(configDir2 / "test2.json", ios::trunc);
fout << greaterPriorityConfig;
fout.close();
fout.open(configDir2 / "test-other.json", ios::trunc);
fout << otherConfig;
fout.close();
diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff();
APSARA_TEST_EQUAL_FATAL(2U, diff.first.mAdded.size());
APSARA_TEST_EQUAL_FATAL(0U, diff.first.mModified.size());
APSARA_TEST_EQUAL_FATAL(1U, diff.first.mRemoved.size());
PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first);
auto allConfigNames = PipelineManagerMock::GetInstance()->GetAllConfigNames();
APSARA_TEST_EQUAL_FATAL(2U + builtinPipelineCnt, allConfigNames.size());
sort(allConfigNames.begin(), allConfigNames.end());
APSARA_TEST_EQUAL_FATAL("test-other", allConfigNames[builtinPipelineCnt]);
APSARA_TEST_EQUAL_FATAL("test2", allConfigNames[builtinPipelineCnt + 1]);
ClearConfig();
}
{
// case: removed -> added, first > second -> first < second
// should not happen
}
{
// case: removed -> added, first < second -> first > second
// should not happen
}
{ // case: removed -> modified, first > second
PrepareConfig();
ofstream fout(configDir1 / "test1.json", ios::trunc);
fout << greaterPriorityConfig;
fout.close();
fout.open(configDir2 / "test2.json", ios::trunc);
fout << lessPriorityConfig;
fout.close();
fout.open(configDir2 / "test-other.json", ios::trunc);
fout << otherConfig;
fout.close();
auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff();
PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first);
APSARA_TEST_EQUAL_FATAL(2U + builtinPipelineCnt,
PipelineManagerMock::GetInstance()->GetAllConfigNames().size());
this_thread::sleep_for(chrono::milliseconds(1));
filesystem::remove(configDir1 / "test1.json");
fout.open(configDir2 / "test2.json", ios::trunc);
fout << modifiedLessPriorityConfig;
fout.close();
fout.open(configDir2 / "test-other.json", ios::trunc);
fout << modifiedOtherConfig;
fout.close();
diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff();
APSARA_TEST_EQUAL_FATAL(1U, diff.first.mAdded.size());
APSARA_TEST_EQUAL_FATAL(1U, diff.first.mModified.size());
APSARA_TEST_EQUAL_FATAL(1U, diff.first.mRemoved.size());
PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first);
auto allConfigNames = PipelineManagerMock::GetInstance()->GetAllConfigNames();
APSARA_TEST_EQUAL_FATAL(2U + builtinPipelineCnt, allConfigNames.size());
sort(allConfigNames.begin(), allConfigNames.end());
APSARA_TEST_EQUAL_FATAL("test-other", allConfigNames[builtinPipelineCnt]);
APSARA_TEST_EQUAL_FATAL("test2", allConfigNames[builtinPipelineCnt + 1]);
ClearConfig();
}
{ // case: removed -> modified, first < second
PrepareConfig();
ofstream fout(configDir1 / "test1.json", ios::trunc);
fout << lessPriorityConfig;
fout.close();
fout.open(configDir2 / "test2.json", ios::trunc);
fout << greaterPriorityConfig;
fout.close();
fout.open(configDir2 / "test-other.json", ios::trunc);
fout << otherConfig;
fout.close();
auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff();
PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first);
APSARA_TEST_EQUAL_FATAL(2U + builtinPipelineCnt,
PipelineManagerMock::GetInstance()->GetAllConfigNames().size());
this_thread::sleep_for(chrono::milliseconds(1));
filesystem::remove(configDir1 / "test1.json");
fout.open(configDir2 / "test2.json", ios::trunc);
fout << modifiedGreaterPriorityConfig;
fout.close();
fout.open(configDir2 / "test-other.json", ios::trunc);
fout << modifiedOtherConfig;
fout.close();
diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff();
APSARA_TEST_EQUAL_FATAL(0U, diff.first.mAdded.size());
APSARA_TEST_EQUAL_FATAL(2U, diff.first.mModified.size());
APSARA_TEST_EQUAL_FATAL(0U, diff.first.mRemoved.size());
PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first);
auto allConfigNames = PipelineManagerMock::GetInstance()->GetAllConfigNames();
APSARA_TEST_EQUAL_FATAL(2U + builtinPipelineCnt, allConfigNames.size());
sort(allConfigNames.begin(), allConfigNames.end());
APSARA_TEST_EQUAL_FATAL("test-other", allConfigNames[builtinPipelineCnt]);
APSARA_TEST_EQUAL_FATAL("test2", allConfigNames[builtinPipelineCnt + 1]);
ClearConfig();
}
{
// case: removed -> modified, first > second -> first < second
// should not happen
}
{
// case: removed -> modified, first < second -> first > second
// should not happen
}
{ // case: removed -> removed, first > second
PrepareConfig();
ofstream fout(configDir1 / "test1.json", ios::trunc);
fout << greaterPriorityConfig;
fout.close();
fout.open(configDir2 / "test2.json", ios::trunc);
fout << lessPriorityConfig;
fout.close();
fout.open(configDir2 / "test-other.json", ios::trunc);
fout << otherConfig;
fout.close();
auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff();
PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first);
APSARA_TEST_EQUAL_FATAL(2U + builtinPipelineCnt,
PipelineManagerMock::GetInstance()->GetAllConfigNames().size());
filesystem::remove(configDir1 / "test1.json");
filesystem::remove(configDir2 / "test2.json");
filesystem::remove(configDir2 / "test-other.json");
diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff();
APSARA_TEST_EQUAL_FATAL(0U, diff.first.mAdded.size());
APSARA_TEST_EQUAL_FATAL(0U, diff.first.mModified.size());
APSARA_TEST_EQUAL_FATAL(2U, diff.first.mRemoved.size());
PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first);
APSARA_TEST_EQUAL_FATAL(0U + builtinPipelineCnt,
PipelineManagerMock::GetInstance()->GetAllConfigNames().size());
ClearConfig();
}
{ // case: removed -> removed, first < second
PrepareConfig();
ofstream fout(configDir1 / "test1.json", ios::trunc);
fout << lessPriorityConfig;
fout.close();
fout.open(configDir2 / "test2.json", ios::trunc);
fout << greaterPriorityConfig;
fout.close();
fout.open(configDir2 / "test-other.json", ios::trunc);
fout << otherConfig;
fout.close();
auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff();
PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first);
APSARA_TEST_EQUAL_FATAL(2U + builtinPipelineCnt,
PipelineManagerMock::GetInstance()->GetAllConfigNames().size());
filesystem::remove(configDir1 / "test1.json");
filesystem::remove(configDir2 / "test2.json");
filesystem::remove(configDir2 / "test-other.json");
diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff();
APSARA_TEST_EQUAL_FATAL(0U, diff.first.mAdded.size());
APSARA_TEST_EQUAL_FATAL(0U, diff.first.mModified.size());
APSARA_TEST_EQUAL_FATAL(2U, diff.first.mRemoved.size());
PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first);
APSARA_TEST_EQUAL_FATAL(0U + builtinPipelineCnt,
PipelineManagerMock::GetInstance()->GetAllConfigNames().size());
ClearConfig();
}
{
// case: removed -> removed, first > second -> first < second
// should not happen
}
{
// case: removed -> removed, first < second -> first > second
// should not happen
}
{ // case: removed -> unchanged, first > second
PrepareConfig();
ofstream fout(configDir1 / "test1.json", ios::trunc);
fout << greaterPriorityConfig;
fout.close();
fout.open(configDir2 / "test2.json", ios::trunc);
fout << lessPriorityConfig;
fout.close();
fout.open(configDir2 / "test-other.json", ios::trunc);
fout << otherConfig;
fout.close();
auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff();
PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first);
APSARA_TEST_EQUAL_FATAL(2U + builtinPipelineCnt,
PipelineManagerMock::GetInstance()->GetAllConfigNames().size());
filesystem::remove(configDir1 / "test1.json");
diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff();
APSARA_TEST_EQUAL_FATAL(1U, diff.first.mAdded.size());
APSARA_TEST_EQUAL_FATAL(0U, diff.first.mModified.size());
APSARA_TEST_EQUAL_FATAL(1U, diff.first.mRemoved.size());
PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first);
auto allConfigNames = PipelineManagerMock::GetInstance()->GetAllConfigNames();
APSARA_TEST_EQUAL_FATAL(2U + builtinPipelineCnt, allConfigNames.size());
sort(allConfigNames.begin(), allConfigNames.end());
APSARA_TEST_EQUAL_FATAL("test-other", allConfigNames[builtinPipelineCnt]);
APSARA_TEST_EQUAL_FATAL("test2", allConfigNames[builtinPipelineCnt + 1]);
ClearConfig();
}
{ // case: removed -> unchanged, first < second
PrepareConfig();
ofstream fout(configDir1 / "test1.json", ios::trunc);
fout << lessPriorityConfig;
fout.close();
fout.open(configDir2 / "test2.json", ios::trunc);
fout << greaterPriorityConfig;
fout.close();
fout.open(configDir2 / "test-other.json", ios::trunc);
fout << otherConfig;
fout.close();
auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff();
PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first);
APSARA_TEST_EQUAL_FATAL(2U + builtinPipelineCnt,
PipelineManagerMock::GetInstance()->GetAllConfigNames().size());
filesystem::remove(configDir1 / "test1.json");
diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff();
APSARA_TEST_EQUAL_FATAL(0U, diff.first.mAdded.size());
APSARA_TEST_EQUAL_FATAL(0U, diff.first.mModified.size());
APSARA_TEST_EQUAL_FATAL(0U, diff.first.mRemoved.size());
PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first);
auto allConfigNames = PipelineManagerMock::GetInstance()->GetAllConfigNames();
APSARA_TEST_EQUAL_FATAL(2U + builtinPipelineCnt, allConfigNames.size());
sort(allConfigNames.begin(), allConfigNames.end());
APSARA_TEST_EQUAL_FATAL("test-other", allConfigNames[builtinPipelineCnt]);
APSARA_TEST_EQUAL_FATAL("test2", allConfigNames[builtinPipelineCnt + 1]);
ClearConfig();
}
{
// case: removed -> unchanged, first > second -> first < second
// should not happen
}
{ // case: removed -> unchanged, first < second -> first > second
// should not happen
}
}
void PipelineConfigWatcherUnittest::TestLoadUnchangedSingletonConfig() {
{ // case: unchanged -> added, first > second
PrepareConfig();
ofstream fout(configDir1 / "test1.json", ios::trunc);
fout << greaterPriorityConfig;
fout.close();
auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff();
size_t builtinPipelineCnt = 0;
#ifdef __ENTERPRISE__
builtinPipelineCnt += EnterpriseConfigProvider::GetInstance()->GetAllBuiltInPipelineConfigs().size();
#endif
PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first);
APSARA_TEST_EQUAL_FATAL(1U + builtinPipelineCnt,
PipelineManagerMock::GetInstance()->GetAllConfigNames().size());
fout.open(configDir2 / "test2.json", ios::trunc);
fout << lessPriorityConfig;
fout.close();
fout.open(configDir2 / "test-other.json", ios::trunc);
fout << otherConfig;
fout.close();
diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff();
APSARA_TEST_EQUAL_FATAL(1U, diff.first.mAdded.size());
APSARA_TEST_EQUAL_FATAL(0U, diff.first.mModified.size());
APSARA_TEST_EQUAL_FATAL(0U, diff.first.mRemoved.size());
PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first);
auto allConfigNames = PipelineManagerMock::GetInstance()->GetAllConfigNames();
APSARA_TEST_EQUAL_FATAL(2U + builtinPipelineCnt, allConfigNames.size());
sort(allConfigNames.begin(), allConfigNames.end());
APSARA_TEST_EQUAL_FATAL("test-other", allConfigNames[builtinPipelineCnt]);
APSARA_TEST_EQUAL_FATAL("test1", allConfigNames[builtinPipelineCnt + 1]);
ClearConfig();
}
{ // case: unchanged -> added, first < second
PrepareConfig();
ofstream fout(configDir1 / "test1.json", ios::trunc);
fout << lessPriorityConfig;
fout.close();
auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff();
PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first);
APSARA_TEST_EQUAL_FATAL(1U + builtinPipelineCnt,
PipelineManagerMock::GetInstance()->GetAllConfigNames().size());
fout.open(configDir2 / "test2.json", ios::trunc);
fout << greaterPriorityConfig;
fout.close();
fout.open(configDir2 / "test-other.json", ios::trunc);
fout << otherConfig;
fout.close();
diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff();
APSARA_TEST_EQUAL_FATAL(2U, diff.first.mAdded.size());
APSARA_TEST_EQUAL_FATAL(0U, diff.first.mModified.size());
APSARA_TEST_EQUAL_FATAL(1U, diff.first.mRemoved.size());
PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first);
auto allConfigNames = PipelineManagerMock::GetInstance()->GetAllConfigNames();
APSARA_TEST_EQUAL_FATAL(2U + builtinPipelineCnt, allConfigNames.size());
sort(allConfigNames.begin(), allConfigNames.end());
APSARA_TEST_EQUAL_FATAL("test-other", allConfigNames[builtinPipelineCnt]);
APSARA_TEST_EQUAL_FATAL("test2", allConfigNames[builtinPipelineCnt + 1]);
ClearConfig();
}
{
// case: unchanged -> added, first > second -> first < second
// should not happen
}
{
// case: unchanged -> added, first < second -> first > second
// should not happen
}
{ // case: unchanged -> modified, first > second
PrepareConfig();
ofstream fout(configDir1 / "test1.json", ios::trunc);
fout << greaterPriorityConfig;
fout.close();
fout.open(configDir2 / "test2.json", ios::trunc);
fout << lessPriorityConfig;
fout.close();
fout.open(configDir2 / "test-other.json", ios::trunc);
fout << otherConfig;
fout.close();
auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff();
PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first);
APSARA_TEST_EQUAL_FATAL(2U + builtinPipelineCnt,
PipelineManagerMock::GetInstance()->GetAllConfigNames().size());
fout.open(configDir2 / "test2.json", ios::trunc);
fout << modifiedLessPriorityConfig;
fout.close();
fout.open(configDir2 / "test-other.json", ios::trunc);
fout << modifiedOtherConfig;
fout.close();
diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff();
APSARA_TEST_EQUAL_FATAL(0U, diff.first.mAdded.size());
APSARA_TEST_EQUAL_FATAL(1U, diff.first.mModified.size());
APSARA_TEST_EQUAL_FATAL(0U, diff.first.mRemoved.size());
PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first);
auto allConfigNames = PipelineManagerMock::GetInstance()->GetAllConfigNames();
APSARA_TEST_EQUAL_FATAL(2U + builtinPipelineCnt, allConfigNames.size());
sort(allConfigNames.begin(), allConfigNames.end());
APSARA_TEST_EQUAL_FATAL("test-other", allConfigNames[builtinPipelineCnt]);
APSARA_TEST_EQUAL_FATAL("test1", allConfigNames[builtinPipelineCnt + 1]);
ClearConfig();
}
{ // case: unchanged -> modified, first < second
PrepareConfig();
ofstream fout(configDir1 / "test1.json", ios::trunc);
fout << lessPriorityConfig;
fout.close();
fout.open(configDir2 / "test2.json", ios::trunc);
fout << greaterPriorityConfig;
fout.close();
fout.open(configDir2 / "test-other.json", ios::trunc);
fout << otherConfig;
fout.close();
auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff();
PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first);
APSARA_TEST_EQUAL_FATAL(2U + builtinPipelineCnt,
PipelineManagerMock::GetInstance()->GetAllConfigNames().size());
fout.open(configDir2 / "test2.json", ios::trunc);
fout << modifiedGreaterPriorityConfig;
fout.close();
fout.open(configDir2 / "test-other.json", ios::trunc);
fout << modifiedOtherConfig;
fout.close();
diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff();
APSARA_TEST_EQUAL_FATAL(0U, diff.first.mAdded.size());
APSARA_TEST_EQUAL_FATAL(2U, diff.first.mModified.size());
APSARA_TEST_EQUAL_FATAL(0U, diff.first.mRemoved.size());
PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first);
auto allConfigNames = PipelineManagerMock::GetInstance()->GetAllConfigNames();
APSARA_TEST_EQUAL_FATAL(2U + builtinPipelineCnt, allConfigNames.size());
sort(allConfigNames.begin(), allConfigNames.end());
APSARA_TEST_EQUAL_FATAL("test-other", allConfigNames[builtinPipelineCnt]);
APSARA_TEST_EQUAL_FATAL("test2", allConfigNames[builtinPipelineCnt + 1]);
ClearConfig();
}
{ // case: unchanged -> modified, first > second -> first < second
PrepareConfig();
ofstream fout(configDir1 / "test1.json", ios::trunc);
fout << lessPriorityConfig;
fout.close();
fout.open(configDir2 / "test2.json", ios::trunc);
fout << lessPriorityConfig;
fout.close();
fout.open(configDir2 / "test-other.json", ios::trunc);
fout << otherConfig;
fout.close();
auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff();
PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first);
APSARA_TEST_EQUAL_FATAL(2U + builtinPipelineCnt,
PipelineManagerMock::GetInstance()->GetAllConfigNames().size());
fout.open(configDir2 / "test2.json", ios::trunc);
fout << greaterPriorityConfig;
fout.close();
fout.open(configDir2 / "test-other.json", ios::trunc);
fout << modifiedOtherConfig;
fout.close();
diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff();
APSARA_TEST_EQUAL_FATAL(1U, diff.first.mAdded.size());
APSARA_TEST_EQUAL_FATAL(1U, diff.first.mModified.size());
APSARA_TEST_EQUAL_FATAL(1U, diff.first.mRemoved.size());
PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first);
auto allConfigNames = PipelineManagerMock::GetInstance()->GetAllConfigNames();
APSARA_TEST_EQUAL_FATAL(2U + builtinPipelineCnt, allConfigNames.size());
sort(allConfigNames.begin(), allConfigNames.end());
APSARA_TEST_EQUAL_FATAL("test-other", allConfigNames[builtinPipelineCnt]);
APSARA_TEST_EQUAL_FATAL("test2", allConfigNames[builtinPipelineCnt + 1]);
ClearConfig();
}
{ // case: unchanged -> modified, first < second -> first > second
PrepareConfig();
ofstream fout(configDir1 / "test3.json", ios::trunc);
fout << greaterPriorityConfig;
fout.close();
fout.open(configDir2 / "test2.json", ios::trunc);
fout << greaterPriorityConfig;
fout.close();
fout.open(configDir2 / "test-other.json", ios::trunc);
fout << otherConfig;
fout.close();
auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff();
PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first);
APSARA_TEST_EQUAL_FATAL(2U + builtinPipelineCnt,
PipelineManagerMock::GetInstance()->GetAllConfigNames().size());
fout.open(configDir2 / "test2.json", ios::trunc);
fout << modifiedLessPriorityConfig;
fout.close();
fout.open(configDir2 / "test-other.json", ios::trunc);
fout << modifiedOtherConfig;
fout.close();
diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff();
APSARA_TEST_EQUAL_FATAL(1U, diff.first.mAdded.size());
APSARA_TEST_EQUAL_FATAL(1U, diff.first.mModified.size());
APSARA_TEST_EQUAL_FATAL(1U, diff.first.mRemoved.size());
PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first);
auto allConfigNames = PipelineManagerMock::GetInstance()->GetAllConfigNames();
APSARA_TEST_EQUAL_FATAL(2U + builtinPipelineCnt, allConfigNames.size());
sort(allConfigNames.begin(), allConfigNames.end());
APSARA_TEST_EQUAL_FATAL("test-other", allConfigNames[builtinPipelineCnt]);
APSARA_TEST_EQUAL_FATAL("test3", allConfigNames[builtinPipelineCnt + 1]);
ClearConfig();
}
{ // case: unchanged -> removed, first > second
PrepareConfig();
ofstream fout(configDir1 / "test1.json", ios::trunc);
fout << greaterPriorityConfig;
fout.close();
fout.open(configDir2 / "test2.json", ios::trunc);
fout << lessPriorityConfig;
fout.close();
fout.open(configDir2 / "test-other.json", ios::trunc);
fout << otherConfig;
fout.close();
auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff();
PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first);
APSARA_TEST_EQUAL_FATAL(2U + builtinPipelineCnt,
PipelineManagerMock::GetInstance()->GetAllConfigNames().size());
filesystem::remove(configDir2 / "test2.json");
filesystem::remove(configDir2 / "test-other.json");
diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff();
APSARA_TEST_EQUAL_FATAL(0U, diff.first.mAdded.size());
APSARA_TEST_EQUAL_FATAL(0U, diff.first.mModified.size());
APSARA_TEST_EQUAL_FATAL(1U, diff.first.mRemoved.size());
PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first);
auto allConfigNames = PipelineManagerMock::GetInstance()->GetAllConfigNames();
APSARA_TEST_EQUAL_FATAL(1U + builtinPipelineCnt, allConfigNames.size());
APSARA_TEST_EQUAL_FATAL("test1", allConfigNames[builtinPipelineCnt]);
ClearConfig();
}
{ // case: unchanged -> removed, first < second
PrepareConfig();
ofstream fout(configDir1 / "test1.json", ios::trunc);
fout << lessPriorityConfig;
fout.close();
fout.open(configDir2 / "test2.json", ios::trunc);
fout << greaterPriorityConfig;
fout.close();
fout.open(configDir2 / "test-other.json", ios::trunc);
fout << otherConfig;
fout.close();
auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff();
PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first);
APSARA_TEST_EQUAL_FATAL(2U + builtinPipelineCnt,
PipelineManagerMock::GetInstance()->GetAllConfigNames().size());
filesystem::remove(configDir2 / "test2.json");
filesystem::remove(configDir2 / "test-other.json");
diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff();
APSARA_TEST_EQUAL_FATAL(1U, diff.first.mAdded.size());
APSARA_TEST_EQUAL_FATAL(0U, diff.first.mModified.size());
APSARA_TEST_EQUAL_FATAL(2U, diff.first.mRemoved.size());
PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first);
auto allConfigNames = PipelineManagerMock::GetInstance()->GetAllConfigNames();
APSARA_TEST_EQUAL_FATAL(1U + builtinPipelineCnt, allConfigNames.size());
APSARA_TEST_EQUAL_FATAL("test1", allConfigNames[0]);
ClearConfig();
}
{
// case: unchanged -> removed, first > second -> first < second
// should not happen
}
{
// case: unchanged -> removed, first < second -> first > second
// should not happen
}
{ // case: unchanged -> unchanged, first > second
PrepareConfig();
ofstream fout(configDir1 / "test1.json", ios::trunc);
fout << greaterPriorityConfig;
fout.close();
fout.open(configDir2 / "test2.json", ios::trunc);
fout << lessPriorityConfig;
fout.close();
fout.open(configDir2 / "test-other.json", ios::trunc);
fout << otherConfig;
fout.close();
auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff();
PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first);
APSARA_TEST_EQUAL_FATAL(2U + builtinPipelineCnt,
PipelineManagerMock::GetInstance()->GetAllConfigNames().size());
diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff();
APSARA_TEST_EQUAL_FATAL(0U, diff.first.mAdded.size());
APSARA_TEST_EQUAL_FATAL(0U, diff.first.mModified.size());
APSARA_TEST_EQUAL_FATAL(0U, diff.first.mRemoved.size());
PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first);
auto allConfigNames = PipelineManagerMock::GetInstance()->GetAllConfigNames();
APSARA_TEST_EQUAL_FATAL(2U + builtinPipelineCnt, allConfigNames.size());
sort(allConfigNames.begin(), allConfigNames.end());
APSARA_TEST_EQUAL_FATAL("test-other", allConfigNames[builtinPipelineCnt]);
APSARA_TEST_EQUAL_FATAL("test1", allConfigNames[builtinPipelineCnt + 1]);
ClearConfig();
}
{ // case: unchanged -> unchanged, first < second
PrepareConfig();
ofstream fout(configDir1 / "test1.json", ios::trunc);
fout << lessPriorityConfig;
fout.close();
fout.open(configDir2 / "test2.json", ios::trunc);
fout << greaterPriorityConfig;
fout.close();
fout.open(configDir2 / "test-other.json", ios::trunc);
fout << otherConfig;
fout.close();
auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff();
PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first);
APSARA_TEST_EQUAL_FATAL(2U + builtinPipelineCnt,
PipelineManagerMock::GetInstance()->GetAllConfigNames().size());
diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff();
APSARA_TEST_EQUAL_FATAL(0U, diff.first.mAdded.size());
APSARA_TEST_EQUAL_FATAL(0U, diff.first.mModified.size());
APSARA_TEST_EQUAL_FATAL(0U, diff.first.mRemoved.size());
PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first);
auto allConfigNames = PipelineManagerMock::GetInstance()->GetAllConfigNames();
APSARA_TEST_EQUAL_FATAL(2U + builtinPipelineCnt, allConfigNames.size());
sort(allConfigNames.begin(), allConfigNames.end());
APSARA_TEST_EQUAL_FATAL("test-other", allConfigNames[builtinPipelineCnt]);
APSARA_TEST_EQUAL_FATAL("test2", allConfigNames[builtinPipelineCnt + 1]);
ClearConfig();
}
{
// case: unchanged -> unchanged, first > second -> first < second
// should not happen
}
{ // case: unchanged -> unchanged, first < second -> first > second
// should not happen
}
}
UNIT_TEST_CASE(PipelineConfigWatcherUnittest, TestLoadAddedSingletonConfig)
UNIT_TEST_CASE(PipelineConfigWatcherUnittest, TestLoadModifiedSingletonConfig)
UNIT_TEST_CASE(PipelineConfigWatcherUnittest, TestLoadRemovedSingletonConfig)
UNIT_TEST_CASE(PipelineConfigWatcherUnittest, TestLoadUnchangedSingletonConfig)
} // namespace logtail
UNIT_TEST_MAIN