in core/unittest/processor/ProcessorParseDelimiterNativeUnittest.cpp [1490:1803]
void ProcessorParseDelimiterNativeUnittest::TestProcessQuote() {
{
std::string inJson = R"({
"events" :
[
{
"contents" :
{
"content" : "'-' 'file0' '947113' '192.168.0.3' '2024-08-15T15:25:24.195264681' 'PUT
/dir/resource.txt
HTTP/2.0' '200' '154' 'go-sdk'"
},
"timestamp" : 12345678901,
"timestampNanosecond": 0,
"type" : 1
}
]
})";
std::string expectJson = R"({
"events": [
{
"contents": {
"1": "-",
"2": "file0",
"3": "947113",
"4": "192.168.0.3",
"5": "2024-08-15T15:25:24.195264681",
"6": "PUT\n/dir/resource.txt\nHTTP/2.0",
"7": "200",
"8": "154",
"9": "go-sdk"
},
"timestamp": 12345678901,
"timestampNanosecond": 0,
"type": 1
}
]
})";
// ProcessorSplitMultilineLogStringNative
{
// make events
auto sourceBuffer = std::make_shared<SourceBuffer>();
PipelineEventGroup eventGroup(sourceBuffer);
eventGroup.FromJsonString(inJson);
// make config
Json::Value config;
config["SourceKey"] = "content";
config["Separator"] = " ";
config["Quote"] = "'";
config["Keys"] = Json::arrayValue;
config["Keys"].append("1");
config["Keys"].append("2");
config["Keys"].append("3");
config["Keys"].append("4");
config["Keys"].append("5");
config["Keys"].append("6");
config["Keys"].append("7");
config["Keys"].append("8");
config["Keys"].append("9");
config["KeepingSourceWhenParseFail"] = false;
config["KeepingSourceWhenParseSucceed"] = false;
config["CopingRawLog"] = false;
config["RenamedSourceKey"] = "__raw__";
config["AllowingShortenedFields"] = true;
config["StartPattern"] = "[a-zA-Z0-9]*";
config["UnmatchedContentTreatment"] = "single_line";
std::string pluginId = "testID";
// run function ProcessorParseDelimiterNative
ProcessorParseDelimiterNative& processorParseDelimiterNative = *(new ProcessorParseDelimiterNative);
ProcessorInstance processorInstance(&processorParseDelimiterNative, getPluginMeta());
APSARA_TEST_TRUE_FATAL(processorInstance.Init(config, mContext));
processorParseDelimiterNative.Process(eventGroup);
// judge result
std::string outJson = eventGroup.ToJsonString();
APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str());
}
}
{
std::string inJson = R"({
"events" :
[
{
"contents" :
{
"content" : " 2023-12-25 1|zdfvzdfv zfdv|zfdvzdfv zfd|fzdvzdfvzdfvz|zfvzfdzv zfdb|zfdvzdfbvzb|zdfvzdfbvzdb|'advfawevaevb|dvzdfvzdbfazdb|zdfvbzdfb '|zdfbvzbszfbsfb
2023-12-25 1|zdfvzdfv zfdv|zfdvzdfv zfd|fzdvzdfvzdfvz|zfvzfdzv zfdb|zfdvzdfbvzb|zdfvzdfbvzdb|'advfawevaevb|dvzdfvzdbfazdb|zdfvbzdfb '|zdfbvzbszfbsfb
2023-12-25 1|zdfvzdfv zfdv|zfdvzdfv zfd|fzdvzdfvzdfvz|zfvzfdzv zfdb|zfdvzdfbvzb|zdfvzdfbvzdb|'advfawevaevb|dvzdfvzdbfazdb|zdfvbzdfb '|zdfbvzbszfbsfb
2023-12-25 1|zdfvzdfv zfdv|zfdvzdfv zfd|fzdvzdfvzdfvz|zfvzfdzv zfdb|zfdvzdfbvzb|zdfvzdfbvzdb|'advfawevaevb|dvzdfvzdbfazdb|zdfvbzdfb '|zdfbvzbszfbsfb"
},
"timestamp" : 12345678901,
"timestampNanosecond": 0,
"type" : 1
}
]
})";
std::string expectJson = R"({
"events": [
{
"contents": {
"1": "2023-12-25 1",
"2": "zdfvzdfv zfdv",
"3": "zfdvzdfv zfd",
"4": "fzdvzdfvzdfvz",
"5": "zfvzfdzv zfdb",
"6": "zfdvzdfbvzb",
"7": "zdfvzdfbvzdb",
"8": "advfawevaevb|dvzdfvzdbfazdb|zdfvbzdfb ",
"9": "zdfbvzbszfbsfb"
},
"timestamp": 12345678901,
"timestampNanosecond": 0,
"type": 1
},
{
"contents": {
"1": "2023-12-25 1",
"2": "zdfvzdfv zfdv",
"3": "zfdvzdfv zfd",
"4": "fzdvzdfvzdfvz",
"5": "zfvzfdzv zfdb",
"6": "zfdvzdfbvzb",
"7": "zdfvzdfbvzdb",
"8": "advfawevaevb|dvzdfvzdbfazdb|zdfvbzdfb ",
"9": "zdfbvzbszfbsfb"
},
"timestamp": 12345678901,
"timestampNanosecond": 0,
"type": 1
},
{
"contents": {
"1": "2023-12-25 1",
"2": "zdfvzdfv zfdv",
"3": "zfdvzdfv zfd",
"4": "fzdvzdfvzdfvz",
"5": "zfvzfdzv zfdb",
"6": "zfdvzdfbvzb",
"7": "zdfvzdfbvzdb",
"8": "advfawevaevb|dvzdfvzdbfazdb|zdfvbzdfb ",
"9": "zdfbvzbszfbsfb"
},
"timestamp": 12345678901,
"timestampNanosecond": 0,
"type": 1
},
{
"contents": {
"1": "2023-12-25 1",
"2": "zdfvzdfv zfdv",
"3": "zfdvzdfv zfd",
"4": "fzdvzdfvzdfvz",
"5": "zfvzfdzv zfdb",
"6": "zfdvzdfbvzb",
"7": "zdfvzdfbvzdb",
"8": "advfawevaevb|dvzdfvzdbfazdb|zdfvbzdfb ",
"9": "zdfbvzbszfbsfb"
},
"timestamp": 12345678901,
"timestampNanosecond": 0,
"type": 1
}
]
})";
// ProcessorSplitMultilineLogStringNative
{
// make events
auto sourceBuffer = std::make_shared<SourceBuffer>();
PipelineEventGroup eventGroup(sourceBuffer);
eventGroup.FromJsonString(inJson);
// make config
Json::Value config;
config["SourceKey"] = "content";
config["Separator"] = "|";
config["Quote"] = "'";
config["Keys"] = Json::arrayValue;
config["Keys"].append("1");
config["Keys"].append("2");
config["Keys"].append("3");
config["Keys"].append("4");
config["Keys"].append("5");
config["Keys"].append("6");
config["Keys"].append("7");
config["Keys"].append("8");
config["Keys"].append("9");
config["Keys"].append("10");
config["Keys"].append("11");
config["KeepingSourceWhenParseFail"] = false;
config["KeepingSourceWhenParseSucceed"] = false;
config["CopingRawLog"] = false;
config["RenamedSourceKey"] = "__raw__";
config["AllowingShortenedFields"] = true;
config["StartPattern"] = "[a-zA-Z0-9]*";
config["UnmatchedContentTreatment"] = "single_line";
// run function ProcessorSplitMultilineLogStringNative
ProcessorSplitMultilineLogStringNative processor;
processor.SetContext(mContext);
processor.SetMetricsRecordRef(ProcessorSplitMultilineLogStringNative::sName, "1");
APSARA_TEST_TRUE_FATAL(processor.Init(config));
processor.Process(eventGroup);
// run function ProcessorParseDelimiterNative
ProcessorParseDelimiterNative& processorParseDelimiterNative = *(new ProcessorParseDelimiterNative);
ProcessorInstance processorInstance(&processorParseDelimiterNative, getPluginMeta());
APSARA_TEST_TRUE_FATAL(processorInstance.Init(config, mContext));
processorParseDelimiterNative.Process(eventGroup);
// judge result
std::string outJson = eventGroup.ToJsonString();
APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str());
}
}
{
// make config
Json::Value config;
config["SourceKey"] = "content";
config["Separator"] = ",";
config["Quote"] = "'";
config["Keys"] = Json::arrayValue;
config["Keys"].append("time");
config["Keys"].append("method");
config["Keys"].append("url");
config["Keys"].append("request_time");
config["KeepingSourceWhenParseFail"] = true;
config["KeepingSourceWhenParseSucceed"] = false;
config["RenamedSourceKey"] = "rawLog";
config["AllowingShortenedFields"] = false;
// make events
auto sourceBuffer = std::make_shared<SourceBuffer>();
PipelineEventGroup eventGroup(sourceBuffer);
std::string inJson = R"({
"events" :
[
{
"contents" :
{
"content" : "2013-10-31 21:03:49,POST,'PutData?Category=YunOsAccountOpLog',0.024"
},
"timestamp" : 12345678901,
"timestampNanosecond": 0,
"type" : 1
},
{
"contents" :
{
"content" : "2013-10-31 21:03:49,POST,'PutData?Category=YunOsAccountOpLog,0.024"
},
"timestamp" : 12345678901,
"timestampNanosecond": 0,
"type" : 1
},
{
"contents" :
{
"content" : "2013-10-31 21:03:49,POST,'PutData?Category=YunOs'AccountOpLog',0.024"
},
"timestamp" : 12345678901,
"timestampNanosecond": 0,
"type" : 1
}
]
})";
eventGroup.FromJsonString(inJson);
// run function
ProcessorParseDelimiterNative& processor = *(new ProcessorParseDelimiterNative);
ProcessorInstance processorInstance(&processor, getPluginMeta());
APSARA_TEST_TRUE_FATAL(processorInstance.Init(config, mContext));
processor.Process(eventGroup);
std::string expectJson = R"({
"events" :
[
{
"contents" :
{
"method": "POST",
"request_time": "0.024",
"time": "2013-10-31 21:03:49",
"url": "PutData?Category=YunOsAccountOpLog"
},
"timestamp" : 12345678901,
"timestampNanosecond": 0,
"type" : 1
},
{
"contents" :
{
"rawLog": "2013-10-31 21:03:49,POST,'PutData?Category=YunOsAccountOpLog,0.024"
},
"timestamp" : 12345678901,
"timestampNanosecond": 0,
"type" : 1
},
{
"contents" :
{
"rawLog": "2013-10-31 21:03:49,POST,'PutData?Category=YunOs'AccountOpLog',0.024"
},
"timestamp" : 12345678901,
"timestampNanosecond": 0,
"type" : 1
}
]
})";
// judge result
std::string outJson = eventGroup.ToJsonString();
APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str());
}
}