in core/unittest/processor/ProcessorMergeMultilineLogNativeUnittest.cpp [274:856]
void ProcessorMergeMultilineLogNativeUnittest::TestProcess() {
// make config
Json::Value config;
config["StartPattern"] = "line.*";
config["EndPattern"] = "endLine.*";
config["MergeType"] = "regex";
config["UnmatchedContentTreatment"] = "single_line";
// make processor
// ProcessorSplitLogStringNative
ProcessorSplitLogStringNative processorSplitLogStringNative;
processorSplitLogStringNative.SetContext(mContext);
APSARA_TEST_TRUE_FATAL(processorSplitLogStringNative.Init(config));
// ProcessorMergeMultilineLogNative
ProcessorMergeMultilineLogNative processorMergeMultilineLogNative;
processorMergeMultilineLogNative.SetContext(mContext);
processorMergeMultilineLogNative.SetMetricsRecordRef(ProcessorMergeMultilineLogNative::sName, "1");
APSARA_TEST_TRUE_FATAL(processorMergeMultilineLogNative.Init(config));
// group为空
{
auto sourceBuffer = std::make_shared<SourceBuffer>();
PipelineEventGroup eventGroup(sourceBuffer);
// run test function
processorSplitLogStringNative.Process(eventGroup);
processorMergeMultilineLogNative.Process(eventGroup);
std::stringstream expectJson;
std::string outJson = eventGroup.ToJsonString();
APSARA_TEST_STREQ("null", CompactJson(outJson).c_str());
}
// 存在不支持的event类型
{ // 某个unmatch 后出现了一个不支持
{
auto sourceBuffer = std::make_shared<SourceBuffer>();
PipelineEventGroup eventGroup(sourceBuffer);
std::string inJson1 = R"({
"events" :
[
{
"contents" :
{
"content" : "line\ncontinue\nendLine\ncontinue"
},
"timestamp" : 12345678901,
"timestampNanosecond" : 0,
"type" : 1
}
]
})";
eventGroup.FromJsonString(inJson1);
eventGroup.AddMetricEvent();
std::string inJson2 = R"({
"events" :
[
{
"contents" :
{
"content" : "line\ncontinue\nendLine\ncontinue"
},
"timestamp" : 12345678901,
"timestampNanosecond" : 0,
"type" : 1
}
]
})";
eventGroup.FromJsonString(inJson2);
// run test function
processorSplitLogStringNative.Process(eventGroup);
processorMergeMultilineLogNative.Process(eventGroup);
std::stringstream expectJson;
expectJson << R"({
"events": [
{
"contents": {
"content": "line\ncontinue\nendLine"
},
"timestamp": 12345678901,
"timestampNanosecond": 0,
"type": 1
},
{
"contents": {
"content": "continue"
},
"timestamp": 12345678901,
"timestampNanosecond": 0,
"type": 1
},
{
"name": "",
"timestamp": 0,
"type": 2,
"value": {
"type": "unknown"
}
},
{
"contents": {
"content": "line"
},
"timestamp": 12345678901,
"timestampNanosecond": 0,
"type": 1
},
{
"contents": {
"content": "continue"
},
"timestamp": 12345678901,
"timestampNanosecond": 0,
"type": 1
},
{
"contents": {
"content": "endLine"
},
"timestamp": 12345678901,
"timestampNanosecond": 0,
"type": 1
},
{
"contents": {
"content": "continue"
},
"timestamp": 12345678901,
"timestampNanosecond": 0,
"type": 1
}
]
})";
std::string outJson = eventGroup.ToJsonString();
APSARA_TEST_STREQ(CompactJson(expectJson.str()).c_str(), CompactJson(outJson).c_str());
}
// 正在匹配过程中 出现了一个不支持
{
auto sourceBuffer = std::make_shared<SourceBuffer>();
PipelineEventGroup eventGroup(sourceBuffer);
std::string inJson = R"({
"events" :
[
{
"contents" :
{
"content" : "line\ncontinue"
},
"timestamp" : 12345678901,
"timestampNanosecond" : 0,
"type" : 1
}
]
})";
eventGroup.FromJsonString(inJson);
eventGroup.AddMetricEvent();
inJson = R"({
"events" :
[
{
"contents" :
{
"content" : "endLine\ncontinue"
},
"timestamp" : 12345678901,
"timestampNanosecond" : 0,
"type" : 1
}
]
})";
eventGroup.FromJsonString(inJson);
// run test function
processorSplitLogStringNative.Process(eventGroup);
processorMergeMultilineLogNative.Process(eventGroup);
std::stringstream expectJson;
expectJson << R"({
"events": [
{
"contents": {
"content": "line"
},
"timestamp": 12345678901,
"timestampNanosecond": 0,
"type": 1
},
{
"contents": {
"content": "continue"
},
"timestamp": 12345678901,
"timestampNanosecond": 0,
"type": 1
},
{
"name": "",
"timestamp": 0,
"type": 2,
"value": {
"type": "unknown"
}
},
{
"contents": {
"content": "endLine"
},
"timestamp": 12345678901,
"timestampNanosecond": 0,
"type": 1
},
{
"contents": {
"content": "continue"
},
"timestamp": 12345678901,
"timestampNanosecond": 0,
"type": 1
}
]
})";
std::string outJson = eventGroup.ToJsonString();
APSARA_TEST_STREQ(CompactJson(expectJson.str()).c_str(), CompactJson(outJson).c_str());
}
// 第一个event就是不支持
{
auto sourceBuffer = std::make_shared<SourceBuffer>();
PipelineEventGroup eventGroup(sourceBuffer);
std::string inJson = R"({
"events" :
[
{
"contents" :
{
"content" : "line1\ncontinue\nline2\ncontinue"
},
"timestamp" : 12345678901,
"timestampNanosecond" : 0,
"type" : 1
}
]
})";
eventGroup.AddMetricEvent();
eventGroup.FromJsonString(inJson);
// run test function
processorSplitLogStringNative.Process(eventGroup);
processorMergeMultilineLogNative.Process(eventGroup);
std::stringstream expectJson;
expectJson << R"({
"events": [
{
"name": "",
"timestamp": 0,
"type": 2,
"value": {
"type": "unknown"
}
},
{
"contents": {
"content": "line1"
},
"timestamp": 12345678901,
"timestampNanosecond": 0,
"type": 1
},
{
"contents": {
"content": "continue"
},
"timestamp": 12345678901,
"timestampNanosecond": 0,
"type": 1
},
{
"contents": {
"content": "line2"
},
"timestamp": 12345678901,
"timestampNanosecond": 0,
"type": 1
},
{
"contents": {
"content": "continue"
},
"timestamp": 12345678901,
"timestampNanosecond": 0,
"type": 1
}
]
})";
std::string outJson = eventGroup.ToJsonString();
APSARA_TEST_STREQ(CompactJson(expectJson.str()).c_str(), CompactJson(outJson).c_str());
}
}
// event group中某条event没有mSourceKey
{
// 某个unmatch 后出现一个没有mSourceKey的event
{
auto sourceBuffer = std::make_shared<SourceBuffer>();
PipelineEventGroup eventGroup(sourceBuffer);
std::string inJson = R"({
"events" :
[
{
"contents" :
{
"content" : "line\ncontinue\nendLine\ncontinue"
},
"timestamp" : 12345678901,
"timestampNanosecond" : 0,
"type" : 1
},
{
"contents" :
{
"aaa" : "line\ncontinue\nendLine\ncontinue"
},
"timestamp" : 12345678901,
"timestampNanosecond" : 0,
"type" : 1
},
{
"contents" :
{
"content" : "line\ncontinue\nendLine\ncontinue"
},
"timestamp" : 12345678901,
"timestampNanosecond" : 0,
"type" : 1
}
]
})";
eventGroup.FromJsonString(inJson);
// run test function
processorSplitLogStringNative.Process(eventGroup);
processorMergeMultilineLogNative.Process(eventGroup);
std::stringstream expectJson;
expectJson << R"({
"events" :
[
{
"contents" :
{
"content" : "line\ncontinue\nendLine"
},
"timestamp" : 12345678901,
"timestampNanosecond" : 0,
"type" : 1
},
{
"contents" :
{
"content" : "continue"
},
"timestamp" : 12345678901,
"timestampNanosecond" : 0,
"type" : 1
},
{
"contents" :
{
"aaa" : "line\ncontinue\nendLine\ncontinue"
},
"timestamp" : 12345678901,
"timestampNanosecond" : 0,
"type" : 1
},
{
"contents": {
"content": "line"
},
"timestamp": 12345678901,
"timestampNanosecond": 0,
"type": 1
},
{
"contents": {
"content": "continue"
},
"timestamp": 12345678901,
"timestampNanosecond": 0,
"type": 1
},
{
"contents": {
"content": "endLine"
},
"timestamp": 12345678901,
"timestampNanosecond": 0,
"type": 1
},
{
"contents": {
"content": "continue"
},
"timestamp": 12345678901,
"timestampNanosecond": 0,
"type": 1
}
]
})";
std::string outJson = eventGroup.ToJsonString();
APSARA_TEST_STREQ(CompactJson(expectJson.str()).c_str(), CompactJson(outJson).c_str());
}
// 正在匹配过程中出现没有mSourceKey的event
{
auto sourceBuffer = std::make_shared<SourceBuffer>();
PipelineEventGroup eventGroup(sourceBuffer);
std::string inJson = R"({
"events" :
[
{
"contents" :
{
"content" : "line\ncontinue"
},
"timestamp" : 12345678901,
"timestampNanosecond" : 0,
"type" : 1
},
{
"contents" :
{
"aaa" : "line\ncontinue\nendLine\ncontinue"
},
"timestamp" : 12345678901,
"timestampNanosecond" : 0,
"type" : 1
},
{
"contents" :
{
"content" : "endLine\ncontinue"
},
"timestamp" : 12345678901,
"timestampNanosecond" : 0,
"type" : 1
}
]
})";
eventGroup.FromJsonString(inJson);
// run test function
processorSplitLogStringNative.Process(eventGroup);
processorMergeMultilineLogNative.Process(eventGroup);
std::stringstream expectJson;
expectJson << R"({
"events" :
[
{
"contents" :
{
"content" : "line"
},
"timestamp" : 12345678901,
"timestampNanosecond" : 0,
"type" : 1
},
{
"contents" :
{
"content" : "continue"
},
"timestamp" : 12345678901,
"timestampNanosecond" : 0,
"type" : 1
},
{
"contents" :
{
"aaa" : "line\ncontinue\nendLine\ncontinue"
},
"timestamp" : 12345678901,
"timestampNanosecond" : 0,
"type" : 1
},
{
"contents" :
{
"content" : "endLine"
},
"timestamp" : 12345678901,
"timestampNanosecond" : 0,
"type" : 1
},
{
"contents" :
{
"content" : "continue"
},
"timestamp" : 12345678901,
"timestampNanosecond" : 0,
"type" : 1
}
]
})";
std::string outJson = eventGroup.ToJsonString();
APSARA_TEST_STREQ(CompactJson(expectJson.str()).c_str(), CompactJson(outJson).c_str());
}
// 第一个就出现没有mSourceKey的event
{
auto sourceBuffer = std::make_shared<SourceBuffer>();
PipelineEventGroup eventGroup(sourceBuffer);
std::string inJson = R"({
"events" :
[
{
"contents" :
{
"aaa" : "line\ncontinue\nendLine\ncontinue"
},
"timestamp" : 12345678901,
"timestampNanosecond" : 0,
"type" : 1
},
{
"contents" :
{
"content" : "line\ncontinue\nendLine\ncontinue"
},
"timestamp" : 12345678901,
"timestampNanosecond" : 0,
"type" : 1
}
]
})";
eventGroup.FromJsonString(inJson);
// run test function
processorSplitLogStringNative.Process(eventGroup);
processorMergeMultilineLogNative.Process(eventGroup);
std::stringstream expectJson;
expectJson << R"({
"events" :
[
{
"contents" :
{
"aaa" : "line\ncontinue\nendLine\ncontinue"
},
"timestamp" : 12345678901,
"timestampNanosecond" : 0,
"type" : 1
},
{
"contents" :
{
"content" : "line"
},
"timestamp" : 12345678901,
"timestampNanosecond" : 0,
"type" : 1
},
{
"contents" :
{
"content" : "continue"
},
"timestamp" : 12345678901,
"timestampNanosecond" : 0,
"type" : 1
},
{
"contents" :
{
"content" : "endLine"
},
"timestamp" : 12345678901,
"timestampNanosecond" : 0,
"type" : 1
},
{
"contents" :
{
"content" : "continue"
},
"timestamp" : 12345678901,
"timestampNanosecond" : 0,
"type" : 1
}
]
})";
std::string outJson = eventGroup.ToJsonString();
APSARA_TEST_STREQ(CompactJson(expectJson.str()).c_str(), CompactJson(outJson).c_str());
}
}
}