def verifyBenchmarkFile()

in benchmarking/frameworks/caffe2/caffe2.py [0:0]

• 93 lines of code
• 18 McCabe index (conditional complexity)

    def verifyBenchmarkFile(self, benchmark, filename, is_post):
        # model is now optional
        if "model" in benchmark:
            model = benchmark["model"]
            assert "files" in model, "Files field is missing in benchmark {}".format(
                filename
            )
            assert "name" in model, "Name field is missing in benchmark {}".format(
                filename
            )
            assert "format" in model, "Format field is missing in benchmark {}".format(
                filename
            )

            for f in model["files"]:
                field = model["files"][f]
                assert (
                    "filename" in field
                ), "Filename is missing in file" + " {} of benchmark {}".format(
                    f, filename
                )
                assert (
                    "location" in field
                ), "Location is missing in file" + " {} of benchmark {}".format(
                    f, filename
                )
                if "md5" not in field:
                    assert not field["location"].startswith(
                        "//"
                    ), "MD5 is missing in file" + " {} of benchmark {}".format(
                        f, filename
                    )

        # tests is mandatory
        assert "tests" in benchmark, "Tests field is missing in benchmark {}".format(
            filename
        )
        tests = benchmark["tests"]

        if is_post:
            assert len(tests) == 1, (
                "After rewrite, only one test in " + "one benchmark."
            )
        else:
            assert len(tests) > 0, "Tests cannot be empty"

        is_generic_test = tests[0]["metric"] == "generic"

        for test in tests:
            assert (
                "metric" in test
            ), "Metric field is missing in " + "benchmark {}".format(filename)

            # no check is needed if the metric is generic
            if is_generic_test:
                assert test["metric"] == "generic", "All tests must be generic"
                continue

            if "iter" not in test:
                test["iter"] = -1
            if "warmup" not in test:
                test["warmup"] = -1

            assert (
                "identifier" in test
            ), "Identifier field is missing in " + "benchmark {}".format(filename)

            if "commands" in test or "command" in test or "arguments" in test:
                continue
            # for backward compatibility purpose
            assert (
                "inputs" in test
            ), "Inputs field is missing in " + "benchmark {}".format(filename)

            num = -1
            for ip_name in test["inputs"]:
                ip = test["inputs"][ip_name]
                assert "shapes" in ip, (
                    "Shapes field is missing in"
                    + " input {}".format(ip_name)
                    + " of benchmark {}".format(filename)
                )
                assert "type" in ip, "Type field is missing in input {}".format(
                    ip_name
                ) + " of benchmark {}".format(filename)
                assert isinstance(ip["shapes"], list), (
                    "Shape field should be a list. However, input "
                    + "{} of benchmark is not.".format(ip_name, filename)
                )

                dims = -1
                for item in ip["shapes"]:
                    assert isinstance(item, list), "Shapes must be a list of list."
                    if dims < 0:
                        dims = len(item)
                    else:
                        assert dims == len(item), (
                            "All shapes of one data must have " + "the same dimension"
                        )

                if num < 0:
                    num = len(ip["shapes"])
                else:
                    assert len(ip["shapes"]) == num, (
                        "The shapes of "
                        + "input {} ".format(ip_name)
                        + "are not of the same dimension in "
                        + "benchmark {}".format(filename)
                    )