def parse_log()

in pyrela/parse_log.py [0:0]

• 61 lines of code
• 12 McCabe index (conditional complexity)

def parse_log(filename, max_epoch):
    lines = open(filename, "r").readlines()
    scores = []
    # perfects = []
    train_rates = []
    buffer_rates = []
    act_rates = []
    # pred_loss = []
    calc_loss = []
    back_prop = []
    sample_update = []
    times = []
    samples = []

    for l in lines:
        if "Speed" in l:
            rates = l.split()
            train_rate = float(rates[2][:-1])
            act_rate = float(rates[4][:-1])
            buffer_rate = float(rates[6][:-1])
            train_rates.append(train_rate)
            act_rates.append(act_rate)
            buffer_rates.append(buffer_rate)
        if "calculating loss" in l:
            t = float(l.split()[3])
            calc_loss.append(t)
        if "backprop & update" in l:
            t = float(l.split()[3])
            back_prop.append(t)
        if "sample data" in l:
            t = float(l.split()[3])
            sample_update.append(t)
        if "updating priority" in l:
            t = float(l.split()[2])
            sample_update[-1] += t
        if "eval score:" in l:
            score = float(l.split()[-1])
            scores.append(score)
        if "total time(train)" in l:
            t = l.split(":")[-1]
            times.append(timestr_to_hour(t))
        if "Total Sample" in l:
            s = numstr_to_num(l.split(" ")[-3][:-1])
            samples.append(s)
        if max_epoch > 0 and len(train_rates) == max_epoch:
            break

    epoch = len(train_rates)
    avg_act_rate = int(np.mean(act_rates[-10:]))
    avg_train_rate = int(np.mean(train_rates[-10:]))
    avg_buffer_rate = int(np.mean(buffer_rates[-10:]))
    return {
        "id": filename,
        "epoch": epoch,
        "act_rate": avg_act_rate,
        "train_rate": avg_train_rate,
        "buffer_rate": avg_buffer_rate,
        "calc_loss": np.mean(calc_loss[-10:]),
        "back_prop": np.mean(back_prop[-10:]),
        "sample": np.mean(sample_update[-10:]),
        "final_score": np.mean(scores[-10:]),
        "scores": scores,
        "times": times[: len(scores)],
        "samples": samples[: len(scores)],
    }