def parse_new_log()

in pyhanabi/tools/parse_log.py [0:0]

• 115 lines of code
• 36 McCabe index (conditional complexity)

def parse_new_log(filename, max_epoch):
    lines = open(filename, "r").readlines()
    times = []
    scores = []
    clone_scores = []
    perfects = []
    train_rate = []
    buffer_rate = []
    act_rate = []
    self_score = []
    aux = []
    aux1 = []
    aux2 = []
    xent_pred = []
    xent_v0 = []
    loss = []

    def get_val_from_line(l):
        a = float(l.split()[3][:-1])
        return a

    for i, l in enumerate(lines):
        if "Time spent =" in l:
            t = float(l.split()[-2])
            if len(times) == 0:
                times.append(t)
            else:
                times.append(times[-1] + t)
        if "Speed" in l:
            split = l.split()
            if "act" in l:
                train = float(split[2][:-1])
                act = float(split[4][:-1])
                buf = float(split[6][:-1])
            else:
                train = float(split[2][:-1])
                act = 0
                buf = float(split[4][:-1])
            train_rate.append(train)
            act_rate.append(act)
            buffer_rate.append(buf)
        if ("eval score:" in l or "eval_score:" in l) and "clone bot" not in l:
            split = l.split()
            score = float(split[4][:-1])
            scores.append(score)
        if "clone bot" in l:
            split = l.split()
            score = float(split[-1])
            clone_scores.append(float(score))
        if "eval score" in l:
            perfect = float(l.split()[6][:-1])
            perfects.append(perfect)
        if "eval: self," in l:
            score = float(l.split()[-3][:-1])
            self_score.append(score)
        if ":aux" in l and "avg" in l:
            aux.append(get_val_from_line(l))
        if ":aux1" in l and "avg" in l:
            aux1.append(get_val_from_line(l))
        if ":aux2" in l and "avg" in l:
            aux2.append(get_val_from_line(l))
        if ":xent_pred" in l and "avg" in l:
            a = float(l.split("avg:")[1].strip().split()[0][:-1])
            xent_pred.append(a)
        if ":xent_v0" in l and "avg" in l:
            a = float(l.split()[3][:-1])
            xent_v0.append(a)
        if ":loss" in l and "avg" in l:
            loss.append(get_val_from_line(l))

        if max_epoch > 0 and (len(scores) == max_epoch or len(xent_pred) == max_epoch):
            break

    if len(scores):
        epoch = len(scores)
    elif len(xent_pred):
        epoch = len(xent_pred)
    elif len(loss):
        epoch = len(loss)
    else:
        epoch = 0

    if len(act_rate):
        avg_act_rate = int(np.mean(act_rate[-10:]))
    else:
        avg_act_rate = 0
    if len(train_rate):
        avg_train_rate = int(np.mean(train_rate[-10:]))
    else:
        avg_train_rate = 0
    if len(buffer_rate):
        avg_buffer_rate = int(np.mean(buffer_rate[-10:]))
    else:
        avg_buffer_rate = 0
    times = [t / 60 / 60 for t in times]

    final_xent_pred = 0 if len(xent_pred) == 0 else np.mean(xent_pred[-10:])
    info = {
        "id": filename,
        "epoch": epoch,
        "act_rate": avg_act_rate,
        "train_rate": avg_train_rate,
        "buffer_rate": avg_buffer_rate,
        "final_score": np.mean(scores[-10:]),
        "scores": scores,
        "clone_scores": clone_scores,
        "final_perfect": np.mean(perfects[-10:]),
        "perfects": perfects,
        "aux": aux,
        "aux1": aux1,
        "aux2": aux2,
        "xent_pred": xent_pred,
        "xent_v0": xent_v0,
        "loss": loss,
        "final_xent_pred": final_xent_pred,
        "final_loss": np.mean(loss[-10:]),
        "times": times,
    }
    if len(self_score) > 0:
        info["selfplay"] = np.mean(self_score[-10:])

    return info