void ProcessorParseDelimiterNativeUnittest::TestAllowingShortenedFields()

in core/unittest/processor/ProcessorParseDelimiterNativeUnittest.cpp [73:387]
269 lines of code
1 McCabe index (conditional complexity)

void ProcessorParseDelimiterNativeUnittest::TestAllowingShortenedFields() {
    // make config
    Json::Value config;
    config["SourceKey"] = "content";
    config["Separator"] = ",";
    config["Quote"] = "'";
    config["Keys"] = Json::arrayValue;
    config["Keys"].append("time");
    config["Keys"].append("method");
    config["Keys"].append("url");
    config["Keys"].append("request_time");
    config["KeepingSourceWhenParseFail"] = true;
    config["KeepingSourceWhenParseSucceed"] = true;
    config["CopingRawLog"] = true;
    config["RenamedSourceKey"] = "rawLog";
    config["AllowingShortenedFields"] = true;
    // make events
    auto sourceBuffer = std::make_shared<SourceBuffer>();
    PipelineEventGroup eventGroup(sourceBuffer);
    std::string inJson = R"({
        "events" :
        [
            {
                "contents" :
                {
                    "content" : "2013-10-31 21:03:49,POST,'PutData?Category=YunOsAccountOpLog',0.024"
                },
                "timestamp" : 12345678901,
                "timestampNanosecond": 0,
                "type" : 1
            },
            {
                "contents" :
                {
                    "content" : "value1"
                },
                "timestamp" : 12345678901,
                "timestampNanosecond": 0,
                "type" : 1
            }
        ]
    })";
    eventGroup.FromJsonString(inJson);
    // run function
    ProcessorParseDelimiterNative& processor = *(new ProcessorParseDelimiterNative);
    ProcessorInstance processorInstance(&processor, getPluginMeta());
    APSARA_TEST_TRUE_FATAL(processorInstance.Init(config, mContext));
    processor.Process(eventGroup);
    std::string expectJson = R"({
        "events": [
            {
                "contents": {
                    "method": "POST",
                    "rawLog": "2013-10-31 21:03:49,POST,'PutData?Category=YunOsAccountOpLog',0.024",
                    "request_time": "0.024",
                    "time": "2013-10-31 21:03:49",
                    "url": "PutData?Category=YunOsAccountOpLog"
                },
                "timestamp": 12345678901,
                "timestampNanosecond": 0,
                "type": 1
            },
            {
                "contents": {
                    "rawLog": "value1",
                    "time": "value1"
                },
                "timestamp": 12345678901,
                "timestampNanosecond": 0,
                "type": 1
            }
        ]
    })";
    // judge result
    std::string outJson = eventGroup.ToJsonString();
    APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str());
    // AllowingShortenedFields false
    {
        std::string inJson = R"({
        "events" :
        [
            {
                "contents" :
                {
                    "content" : "123@@45
012@@34"
                },
                "timestamp" : 12345678901,
                "timestampNanosecond": 0,
                "type" : 1
            }
        ]
        })";

        std::string expectJson = R"({
            "events": [
                {
                    "contents": {
                        "__raw__": "123@@45"
                    },
                    "timestamp": 12345678901,
                    "timestampNanosecond": 0,
                    "type": 1
                },
                {
                    "contents": {
                        "__raw__": "012@@34"
                    },
                    "timestamp": 12345678901,
                    "timestampNanosecond": 0,
                    "type": 1
                }
            ]
        })";

        // ProcessorSplitLogStringNative
        {
            // make events
            auto sourceBuffer = std::make_shared<SourceBuffer>();
            PipelineEventGroup eventGroup(sourceBuffer);
            eventGroup.FromJsonString(inJson);

            // make config
            Json::Value config;
            config["SourceKey"] = "content";
            config["Separator"] = "@@";
            config["Quote"] = "'";
            config["Keys"] = Json::arrayValue;
            config["Keys"].append("a");
            config["Keys"].append("b");
            config["Keys"].append("c");
            config["KeepingSourceWhenParseFail"] = true;
            config["KeepingSourceWhenParseSucceed"] = false;
            config["CopingRawLog"] = false;
            config["RenamedSourceKey"] = "__raw__";
            config["AllowingShortenedFields"] = false;
            config["SplitChar"] = '\n';

            // run function ProcessorSplitLogStringNative
            ProcessorSplitLogStringNative processor;
            processor.SetContext(mContext);
            APSARA_TEST_TRUE_FATAL(processor.Init(config));
            processor.Process(eventGroup);

            // run function ProcessorParseDelimiterNative
            ProcessorParseDelimiterNative& processorParseDelimiterNative = *(new ProcessorParseDelimiterNative);
            ProcessorInstance processorInstance(&processorParseDelimiterNative, getPluginMeta());
            APSARA_TEST_TRUE_FATAL(processorInstance.Init(config, mContext));
            processorParseDelimiterNative.Process(eventGroup);

            // judge result
            std::string outJson = eventGroup.ToJsonString();
            APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str());
        }
        // ProcessorSplitMultilineLogStringNative
        {
            // make events
            auto sourceBuffer = std::make_shared<SourceBuffer>();
            PipelineEventGroup eventGroup(sourceBuffer);
            eventGroup.FromJsonString(inJson);

            // make config
            Json::Value config;
            config["SourceKey"] = "content";
            config["Separator"] = "@@";
            config["Quote"] = "'";
            config["Keys"] = Json::arrayValue;
            config["Keys"].append("a");
            config["Keys"].append("b");
            config["Keys"].append("c");
            config["KeepingSourceWhenParseFail"] = true;
            config["KeepingSourceWhenParseSucceed"] = false;
            config["CopingRawLog"] = false;
            config["RenamedSourceKey"] = "__raw__";
            config["AllowingShortenedFields"] = false;
            config["StartPattern"] = "[a-zA-Z0-9]*";
            config["UnmatchedContentTreatment"] = "single_line";

            // run function ProcessorSplitMultilineLogStringNative
            ProcessorSplitMultilineLogStringNative processor;
            processor.SetContext(mContext);
            processor.SetMetricsRecordRef(ProcessorSplitMultilineLogStringNative::sName, "1");
            APSARA_TEST_TRUE_FATAL(processor.Init(config));
            processor.Process(eventGroup);

            // run function ProcessorParseDelimiterNative
            ProcessorParseDelimiterNative& processorParseDelimiterNative = *(new ProcessorParseDelimiterNative);
            ProcessorInstance processorInstance(&processorParseDelimiterNative, getPluginMeta());
            APSARA_TEST_TRUE_FATAL(processorInstance.Init(config, mContext));
            processorParseDelimiterNative.Process(eventGroup);

            // judge result
            std::string outJson = eventGroup.ToJsonString();
            APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str());
        }
    }
    // AllowingShortenedFields true
    {
        std::string inJson = R"({
        "events" :
        [
            {
                "contents" :
                {
                    "content" : "123@@45
012@@34"
                },
                "timestamp" : 12345678901,
                "timestampNanosecond": 0,
                "type" : 1
            }
        ]
        })";

        std::string expectJson = R"({
            "events": [
                {
                    "contents": {
                        "a": "123",
                        "b": "45"
                    },
                    "timestamp": 12345678901,
                    "timestampNanosecond": 0,
                    "type": 1
                },
                {
                    "contents": {
                        "a": "012",
                        "b": "34"
                    },
                    "timestamp": 12345678901,
                    "timestampNanosecond": 0,
                    "type": 1
                }
            ]
        })";

        // ProcessorSplitLogStringNative
        {
            // make events
            auto sourceBuffer = std::make_shared<SourceBuffer>();
            PipelineEventGroup eventGroup(sourceBuffer);
            eventGroup.FromJsonString(inJson);

            // make config
            Json::Value config;
            config["SourceKey"] = "content";
            config["Separator"] = "@@";
            config["Quote"] = "'";
            config["Keys"] = Json::arrayValue;
            config["Keys"].append("a");
            config["Keys"].append("b");
            config["Keys"].append("c");
            config["KeepingSourceWhenParseFail"] = true;
            config["KeepingSourceWhenParseSucceed"] = false;
            config["CopingRawLog"] = false;
            config["RenamedSourceKey"] = "__raw__";
            config["AllowingShortenedFields"] = true;
            config["SplitChar"] = '\n';

            // run function ProcessorSplitLogStringNative
            ProcessorSplitLogStringNative processor;
            processor.SetContext(mContext);
            APSARA_TEST_TRUE_FATAL(processor.Init(config));
            processor.Process(eventGroup);
            // run function ProcessorParseDelimiterNative
            ProcessorParseDelimiterNative& processorParseDelimiterNative = *(new ProcessorParseDelimiterNative);
            ProcessorInstance processorInstance(&processorParseDelimiterNative, getPluginMeta());
            APSARA_TEST_TRUE_FATAL(processorInstance.Init(config, mContext));
            processorParseDelimiterNative.Process(eventGroup);
            // judge result
            std::string outJson = eventGroup.ToJsonString();
            APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str());
        }
        // ProcessorSplitMultilineLogStringNative
        {
            // make events
            auto sourceBuffer = std::make_shared<SourceBuffer>();
            PipelineEventGroup eventGroup(sourceBuffer);
            eventGroup.FromJsonString(inJson);

            // make config
            Json::Value config;
            config["SourceKey"] = "content";
            config["Separator"] = "@@";
            config["Quote"] = "'";
            config["Keys"] = Json::arrayValue;
            config["Keys"].append("a");
            config["Keys"].append("b");
            config["Keys"].append("c");
            config["KeepingSourceWhenParseFail"] = true;
            config["KeepingSourceWhenParseSucceed"] = false;
            config["CopingRawLog"] = false;
            config["RenamedSourceKey"] = "__raw__";
            config["AllowingShortenedFields"] = true;
            config["StartPattern"] = "[a-zA-Z0-9]*";
            config["UnmatchedContentTreatment"] = "single_line";

            // run function ProcessorSplitMultilineLogStringNative
            ProcessorSplitMultilineLogStringNative processor;
            processor.SetContext(mContext);
            processor.SetMetricsRecordRef(ProcessorSplitMultilineLogStringNative::sName, "1");
            APSARA_TEST_TRUE_FATAL(processor.Init(config));
            processor.Process(eventGroup);
            // run function ProcessorParseDelimiterNative
            ProcessorParseDelimiterNative& processorParseDelimiterNative = *(new ProcessorParseDelimiterNative);
            ProcessorInstance processorInstance(&processorParseDelimiterNative, getPluginMeta());
            APSARA_TEST_TRUE_FATAL(processorInstance.Init(config, mContext));
            processorParseDelimiterNative.Process(eventGroup);
            // judge result
            std::string outJson = eventGroup.ToJsonString();
            APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str());
        }
    }
}