in core/unittest/processor/ProcessorParseDelimiterNativeUnittest.cpp [1025:1408]
void ProcessorParseDelimiterNativeUnittest::TestMultipleLinesWithProcessorMergeMultilineLogNative() {
// case < field
{
std::string inJson = R"({
"events" :
[
{
"contents" :
{
"content" : "123@@456
012@@345"
},
"timestamp" : 12345678901,
"timestampNanosecond": 0,
"type" : 1
}
]
})";
std::string expectJson = R"({
"events": [
{
"contents": {
"__raw__": "123@@456"
},
"timestamp": 12345678901,
"timestampNanosecond": 0,
"type": 1
},
{
"contents": {
"__raw__": "012@@345"
},
"timestamp": 12345678901,
"timestampNanosecond": 0,
"type": 1
}
]
})";
// ProcessorSplitLogStringNative
{
// make events
auto sourceBuffer = std::make_shared<SourceBuffer>();
PipelineEventGroup eventGroup(sourceBuffer);
eventGroup.FromJsonString(inJson);
// make config
Json::Value config;
config["SourceKey"] = "content";
config["Separator"] = "@@";
config["Quote"] = "'";
config["Keys"] = Json::arrayValue;
config["Keys"].append("a");
config["Keys"].append("b");
config["Keys"].append("c");
config["KeepingSourceWhenParseFail"] = true;
config["KeepingSourceWhenParseSucceed"] = false;
config["CopingRawLog"] = false;
config["RenamedSourceKey"] = "__raw__";
config["AllowingShortenedFields"] = false;
config["SplitChar"] = '\n';
// run function ProcessorSplitLogStringNative
ProcessorSplitLogStringNative processor;
processor.SetContext(mContext);
APSARA_TEST_TRUE_FATAL(processor.Init(config));
processor.Process(eventGroup);
// run function ProcessorParseDelimiterNative
ProcessorParseDelimiterNative& processorParseDelimiterNative = *(new ProcessorParseDelimiterNative);
ProcessorInstance processorInstance(&processorParseDelimiterNative, getPluginMeta());
APSARA_TEST_TRUE_FATAL(processorInstance.Init(config, mContext));
processorParseDelimiterNative.Process(eventGroup);
// judge result
std::string outJson = eventGroup.ToJsonString();
APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str());
}
// ProcessorMergeMultilineLogNative
{
// make events
auto sourceBuffer = std::make_shared<SourceBuffer>();
PipelineEventGroup eventGroup(sourceBuffer);
eventGroup.FromJsonString(inJson);
// make config
Json::Value config;
config["SourceKey"] = "content";
config["Separator"] = "@@";
config["Quote"] = "'";
config["Keys"] = Json::arrayValue;
config["Keys"].append("a");
config["Keys"].append("b");
config["Keys"].append("c");
config["KeepingSourceWhenParseFail"] = true;
config["KeepingSourceWhenParseSucceed"] = false;
config["CopingRawLog"] = false;
config["RenamedSourceKey"] = "__raw__";
config["AllowingShortenedFields"] = false;
config["StartPattern"] = "[123|012].*";
config["MergeType"] = "regex";
config["UnmatchedContentTreatment"] = "single_line";
// run function ProcessorSplitLogStringNative
ProcessorSplitLogStringNative processorSplitLogStringNative;
processorSplitLogStringNative.SetContext(mContext);
APSARA_TEST_TRUE_FATAL(processorSplitLogStringNative.Init(config));
processorSplitLogStringNative.Process(eventGroup);
// run function ProcessorMergeMultilineLogNative
ProcessorMergeMultilineLogNative processorMergeMultilineLogNative;
processorMergeMultilineLogNative.SetContext(mContext);
processorMergeMultilineLogNative.SetMetricsRecordRef(ProcessorMergeMultilineLogNative::sName, "1");
APSARA_TEST_TRUE_FATAL(processorMergeMultilineLogNative.Init(config));
processorMergeMultilineLogNative.Process(eventGroup);
// run function ProcessorParseDelimiterNative
ProcessorParseDelimiterNative& processorParseDelimiterNative = *(new ProcessorParseDelimiterNative);
ProcessorInstance processorInstance(&processorParseDelimiterNative, getPluginMeta());
APSARA_TEST_TRUE_FATAL(processorInstance.Init(config, mContext));
processorParseDelimiterNative.Process(eventGroup);
// judge result
std::string outJson = eventGroup.ToJsonString();
APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str());
}
}
// case > field
{
std::string inJson = R"({
"events" :
[
{
"contents" :
{
"content" : "123@@456@@789
012@@345@@678"
},
"timestamp" : 12345678901,
"timestampNanosecond": 0,
"type" : 1
}
]
})";
std::string expectJson = R"({
"events": [
{
"contents": {
"__column2__": "789",
"a": "123",
"b": "456"
},
"timestamp": 12345678901,
"timestampNanosecond": 0,
"type": 1
},
{
"contents": {
"__column2__": "678",
"a": "012",
"b": "345"
},
"timestamp": 12345678901,
"timestampNanosecond": 0,
"type": 1
}
]
})";
// ProcessorSplitLogStringNative
{
// make events
auto sourceBuffer = std::make_shared<SourceBuffer>();
PipelineEventGroup eventGroup(sourceBuffer);
eventGroup.FromJsonString(inJson);
// make config
Json::Value config;
config["SourceKey"] = "content";
config["Separator"] = "@@";
config["Quote"] = "'";
config["Keys"] = Json::arrayValue;
config["Keys"].append("a");
config["Keys"].append("b");
config["KeepingSourceWhenParseFail"] = true;
config["KeepingSourceWhenParseSucceed"] = false;
config["CopingRawLog"] = false;
config["RenamedSourceKey"] = "__raw__";
config["AllowingShortenedFields"] = false;
config["SplitChar"] = '\n';
// run function ProcessorSplitLogStringNative
ProcessorSplitLogStringNative processor;
processor.SetContext(mContext);
APSARA_TEST_TRUE_FATAL(processor.Init(config));
processor.Process(eventGroup);
// run function ProcessorParseDelimiterNative
ProcessorParseDelimiterNative& processorParseDelimiterNative = *(new ProcessorParseDelimiterNative);
ProcessorInstance processorInstance(&processorParseDelimiterNative, getPluginMeta());
APSARA_TEST_TRUE_FATAL(processorInstance.Init(config, mContext));
processorParseDelimiterNative.Process(eventGroup);
// judge result
std::string outJson = eventGroup.ToJsonString();
APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str());
}
// ProcessorMergeMultilineLogNative
{
// make events
auto sourceBuffer = std::make_shared<SourceBuffer>();
PipelineEventGroup eventGroup(sourceBuffer);
eventGroup.FromJsonString(inJson);
// make config
Json::Value config;
config["SourceKey"] = "content";
config["Separator"] = "@@";
config["Quote"] = "'";
config["Keys"] = Json::arrayValue;
config["Keys"].append("a");
config["Keys"].append("b");
config["KeepingSourceWhenParseFail"] = true;
config["KeepingSourceWhenParseSucceed"] = false;
config["CopingRawLog"] = false;
config["RenamedSourceKey"] = "__raw__";
config["AllowingShortenedFields"] = false;
config["StartPattern"] = "[123|012].*";
config["MergeType"] = "regex";
config["UnmatchedContentTreatment"] = "single_line";
// run function ProcessorSplitLogStringNative
ProcessorSplitLogStringNative processorSplitLogStringNative;
processorSplitLogStringNative.SetContext(mContext);
APSARA_TEST_TRUE_FATAL(processorSplitLogStringNative.Init(config));
processorSplitLogStringNative.Process(eventGroup);
// run function ProcessorMergeMultilineLogNative
ProcessorMergeMultilineLogNative processorMergeMultilineLogNative;
processorMergeMultilineLogNative.SetContext(mContext);
processorMergeMultilineLogNative.SetMetricsRecordRef(ProcessorMergeMultilineLogNative::sName, "1");
APSARA_TEST_TRUE_FATAL(processorMergeMultilineLogNative.Init(config));
processorMergeMultilineLogNative.Process(eventGroup);
// run function ProcessorParseDelimiterNative
ProcessorParseDelimiterNative& processorParseDelimiterNative = *(new ProcessorParseDelimiterNative);
ProcessorInstance processorInstance(&processorParseDelimiterNative, getPluginMeta());
APSARA_TEST_TRUE_FATAL(processorInstance.Init(config, mContext));
processorParseDelimiterNative.Process(eventGroup);
// judge result
std::string outJson = eventGroup.ToJsonString();
APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str());
}
}
// case = field
{
std::string inJson = R"({
"events" :
[
{
"contents" :
{
"content" : "123@@456@@789
012@@345@@678"
},
"timestamp" : 12345678901,
"timestampNanosecond": 0,
"type" : 1
}
]
})";
std::string expectJson = R"({
"events" :
[
{
"contents" :
{
"a": "123",
"b": "456",
"c": "789"
},
"timestamp" : 12345678901,
"timestampNanosecond": 0,
"type" : 1
},
{
"contents" :
{
"a": "012",
"b": "345",
"c": "678"
},
"timestamp" : 12345678901,
"timestampNanosecond": 0,
"type" : 1
}
]
})";
// ProcessorSplitLogStringNative
{
// make events
auto sourceBuffer = std::make_shared<SourceBuffer>();
PipelineEventGroup eventGroup(sourceBuffer);
eventGroup.FromJsonString(inJson);
// make config
Json::Value config;
config["SourceKey"] = "content";
config["Separator"] = "@@";
config["Quote"] = "'";
config["Keys"] = Json::arrayValue;
config["Keys"].append("a");
config["Keys"].append("b");
config["Keys"].append("c");
config["KeepingSourceWhenParseFail"] = true;
config["KeepingSourceWhenParseSucceed"] = false;
config["CopingRawLog"] = false;
config["RenamedSourceKey"] = "__raw__";
config["AllowingShortenedFields"] = false;
config["SplitChar"] = '\n';
// run function ProcessorSplitLogStringNative
ProcessorSplitLogStringNative processor;
processor.SetContext(mContext);
APSARA_TEST_TRUE_FATAL(processor.Init(config));
processor.Process(eventGroup);
// run function ProcessorParseDelimiterNative
ProcessorParseDelimiterNative& processorParseDelimiterNative = *(new ProcessorParseDelimiterNative);
ProcessorInstance processorInstance(&processorParseDelimiterNative, getPluginMeta());
APSARA_TEST_TRUE_FATAL(processorInstance.Init(config, mContext));
processorParseDelimiterNative.Process(eventGroup);
// judge result
std::string outJson = eventGroup.ToJsonString();
APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str());
}
// ProcessorMergeMultilineLogNative
{
// make events
auto sourceBuffer = std::make_shared<SourceBuffer>();
PipelineEventGroup eventGroup(sourceBuffer);
eventGroup.FromJsonString(inJson);
// make config
Json::Value config;
config["SourceKey"] = "content";
config["Separator"] = "@@";
config["Quote"] = "'";
config["Keys"] = Json::arrayValue;
config["Keys"].append("a");
config["Keys"].append("b");
config["Keys"].append("c");
config["KeepingSourceWhenParseFail"] = true;
config["KeepingSourceWhenParseSucceed"] = false;
config["CopingRawLog"] = false;
config["RenamedSourceKey"] = "__raw__";
config["AllowingShortenedFields"] = false;
config["StartPattern"] = "[123|012].*";
config["MergeType"] = "regex";
config["UnmatchedContentTreatment"] = "single_line";
// run function ProcessorSplitLogStringNative
ProcessorSplitLogStringNative processorSplitLogStringNative;
processorSplitLogStringNative.SetContext(mContext);
APSARA_TEST_TRUE_FATAL(processorSplitLogStringNative.Init(config));
processorSplitLogStringNative.Process(eventGroup);
// run function ProcessorMergeMultilineLogNative
ProcessorMergeMultilineLogNative processorMergeMultilineLogNative;
processorMergeMultilineLogNative.SetContext(mContext);
processorMergeMultilineLogNative.SetMetricsRecordRef(ProcessorMergeMultilineLogNative::sName, "1");
APSARA_TEST_TRUE_FATAL(processorMergeMultilineLogNative.Init(config));
processorMergeMultilineLogNative.Process(eventGroup);
// run function ProcessorParseDelimiterNative
ProcessorParseDelimiterNative& processorParseDelimiterNative = *(new ProcessorParseDelimiterNative);
ProcessorInstance processorInstance(&processorParseDelimiterNative, getPluginMeta());
APSARA_TEST_TRUE_FATAL(processorInstance.Init(config, mContext));
processorParseDelimiterNative.Process(eventGroup);
// judge result
std::string outJson = eventGroup.ToJsonString();
APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str());
}
}
}