benchmarking/frameworks/glow/glow.py

#!/usr/bin/env python

##############################################################################
# Copyright 2017-present, Facebook, Inc.
# All rights reserved.
#
# This source code is licensed under the license found in the
# LICENSE file in the root directory of this source tree.
##############################################################################

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import json
import os
import re
from collections import defaultdict

from frameworks.framework_base import FrameworkBase
from six import string_types


class GlowFramework(FrameworkBase):
    def __init__(self, tempdir, args):
        super(GlowFramework, self).__init__(args)
        self.tempdir = os.path.join(tempdir, self.getName())
        os.makedirs(self.tempdir, 0o777)

    def getName(self):
        return "glow"

    def runBenchmark(self, info, benchmark, platform):
        output, output_files = super(GlowFramework, self).runBenchmark(
            info, benchmark, platform
        )
        return output, output_files

    def composeRunCommand(
        self,
        commands,
        platform,
        programs,
        model,
        test,
        model_files,
        input_files,
        output_files,
        shared_libs,
        preprocess_files=None,
        main_command=False,
    ):
        cmds = super(GlowFramework, self).composeRunCommand(
            commands,
            platform,
            programs,
            model,
            test,
            model_files,
            input_files,
            output_files,
            shared_libs,
            preprocess_files,
            main_command,
        )
        return cmds

    def runOnPlatform(self, total_num, cmd, platform, platform_args, converter):
        output, meta = platform.runBenchmark(cmd, platform_args=platform_args)
        results = {}
        self._maybeAddJsonOutput(output, results)
        self._maybeAddTraceOutput(platform, results)
        self._maybeAddBenchSummary(output, results)
        self._maybeAddNetRunnerStats(output, results)
        self._maybeNetRunner(output, results)
        self._maybeRepro(output, results)
        results["meta"] = meta
        return results

    def _maybeRepro(self, output, results):
        if output is None:
            return False
        rows = output
        if isinstance(output, string_types):
            rows = output.split("\n")
        i = 0
        while i < len(rows):
            if "Total inference duration (ms): " in rows[i]:
                total_inferece_time = float(
                    rows[i].split("Total inference duration (ms): ")[1]
                )
                self._addOrAppendResult(
                    results,
                    "Total inference duration",
                    total_inferece_time,
                    {
                        "type": "NET",
                        "metric": "Total inference duration",
                        "unit": "ms",
                        "values": [],
                    },
                )
            if "Avg inference duration (ms): " in rows[i]:
                avg_inference_time = float(
                    rows[i].split("Avg inference duration (ms): ")[1]
                )
                self._addOrAppendResult(
                    results,
                    "Avg inference duration",
                    avg_inference_time,
                    {
                        "type": "NET",
                        "metric": "Avg inference duration",
                        "unit": "scalar",
                        "values": [],
                    },
                )
            if "Avg inference per second: " in rows[i]:
                avg_inference_per_second = float(
                    rows[i].split("Avg inference per second: ")[1]
                )
                self._addOrAppendResult(
                    results,
                    "Avg inference per second",
                    avg_inference_per_second,
                    {
                        "type": "NET",
                        "metric": "Avg inference per second",
                        "unit": "scalar",
                        "values": [],
                    },
                )
            i += 1

    def _maybeNetRunner(self, output, results):
        if output is None:
            return False
        rows = output
        if isinstance(output, string_types):
            rows = output.split("\n")
        i = 0
        while i < len(rows):
            match = re.search(r"(.*)latency per (.*) \[(.*)\]:", rows[i])
            if match:
                if match.group(3) == "glow":
                    mtype = "NET"
                else:
                    mtype = "SECONDARY"
                name = match.group(3)
                latency_kind = match.group(2)
                card = match.group(1)
                if card:
                    latency_kind = "card " + latency_kind
                i += 1
                while i < len(rows) and "latency per" not in rows[i].lower():
                    match = re.search(
                        r".*latency\((.*)\): p(.*): (.*)", rows[i].lower()
                    )
                    if match:
                        unit = match.group(1)
                        percentile = "p" + match.group(2)
                        value = float(match.group(3))

                        self._addOrAppendResult(
                            results,
                            " ".join(
                                [mtype, name, "net_runner", latency_kind, percentile]
                            ),
                            value,
                            {
                                "type": mtype,
                                "metric": " ".join(
                                    [name, "net_runner", latency_kind, percentile]
                                ),
                                "unit": unit,
                                "values": [],
                            },
                        )
                    i += 1
            else:
                i += 1

        i = 0
        while i < len(rows):
            match = re.search(r"(.*): (.*) vs (.*)\((.*)\)", rows[i])
            if match:
                test_impls1, test_impls2 = sorted([match.group(2), match.group(3)])
                i += 1
                while i < len(rows) and "abs error" in rows[i].lower():
                    match = re.search(r".*abs error p(.*): (.*)", rows[i].lower())
                    if match:
                        percentile = "p" + match.group(1)
                        value = float(match.group(2))

                        self._addOrAppendResult(
                            results,
                            " ".join(
                                [
                                    "NET",
                                    test_impls1,
                                    "vs",
                                    test_impls2,
                                    "abs error",
                                    percentile,
                                ]
                            ),
                            value,
                            {
                                "type": "NET",
                                "metric": " ".join(
                                    [
                                        test_impls1,
                                        "vs",
                                        test_impls2,
                                        "abs error",
                                        percentile,
                                    ]
                                ),
                                "unit": "scalar",
                                "values": [],
                            },
                        )
                    i += 1

            else:
                i += 1

    def _maybeAddJsonOutput(self, output, results):
        if output is None:
            return False
        rows = output
        if isinstance(output, string_types):
            rows = output.split("\n")
        i = 0
        while i < len(rows):
            try:
                parsed = json.loads(rows[i])
                results[parsed["type"] + " " + parsed["metric"]] = parsed
            except json.JSONDecodeError:
                pass
            i += 1

    def _addOrAppendResult(self, results, key, value, record):
        if key not in results.keys():
            results[key] = record
        results[key]["values"].append(value)

    def _maybeAddBenchSummary(self, output, results):
        existingMaps = {
            "AddBench": (10, 11),
            "BatchGemmBench": (12, 13),
            "GemmBench": (12, 13),
            "GemmParallelBench": (11, 12),
            "SLSBench": (10, 11),
            "TransposeBench": (11, 12),
        }

        fieldMap = defaultdict(lambda: (10, 11))
        for k in existingMaps:
            fieldMap[k] = existingMaps[k]

        if output is None:
            return False
        rows = output
        if isinstance(output, string_types):
            rows = output.split("\n")
        i = 0
        while i < len(rows):
            try:
                fields = rows[i].split(",")
                if fields[0] == "BenchResult":
                    benchName = fields[1]

                    runtimeRecord = {
                        "type": "NET",
                        "metric": "{}:runtime".format(benchName),
                        "unit": "second",
                        "values": [],
                    }
                    throughputRecord = {
                        "type": "SECONDARY",
                        "metric": "{}:throughput".format(benchName),
                        "unit": "Gb/second",
                        "values": [],
                    }

                    self._addOrAppendResult(
                        results,
                        "NET {}:runtime".format(benchName),
                        float(fields[fieldMap[benchName][0]]),
                        runtimeRecord,
                    )
                    self._addOrAppendResult(
                        results,
                        "SECONDARY {}:throughput".format(benchName),
                        float(fields[fieldMap[benchName][1]]),
                        throughputRecord,
                    )

            except IndexError:
                pass
            except ValueError:
                pass
            i += 1

    def _maybeAddNetRunnerStats(self, output, results):
        if output is None:
            return False
        rows = output
        if isinstance(output, string_types):
            rows = output.split("\n")
        i = 0
        while i < len(rows):
            m = re.match(
                r"^individual inference latency \[(\w+)\]: ([0-9]+) us$", rows[i]
            )
            if m:
                if m.groups()[0] == "glow":
                    mtype = "NET"
                else:
                    mtype = "SECONDARY"
                self._addOrAppendResult(
                    results,
                    mtype + " " + m.groups()[0] + " net_runner inference",
                    int(m.groups()[1]),
                    {
                        "type": mtype,
                        "metric": m.groups()[0] + " net_runner inference",
                        "unit": "microsecond",
                        "values": [],
                    },
                )
            i += 1

    def _maybeAddTraceOutput(self, platform, results):
        traceFile = os.path.join(platform.getOutputDir(), "trace")
        if not os.path.exists(traceFile):
            return
        with open(traceFile, "r") as fp:
            line = fp.readline()
            while line:
                try:
                    parsed = json.loads(line.rstrip(", \n\t"))
                    metric = parsed["name"]
                    if not metric:
                        raise ValueError("empty metric")
                    if metric == "inference_e2e":
                        mtype = "NET"
                    else:
                        mtype = "SECONDARY"
                    key = mtype + " " + metric
                    self._addOrAppendResult(
                        results,
                        key,
                        parsed["dur"],
                        {
                            "type": mtype,
                            "metric": metric,
                            "unit": "microsecond",
                            "values": [],
                        },
                    )
                except json.JSONDecodeError:
                    pass
                except KeyError:
                    pass
                except ValueError:
                    pass
                line = fp.readline()