scripts/qlog_parser.py

import argparse
import os
import sys
import json
import re
import datetime

connectivity_event_list_ = ["server_listening",      "connection_started",  "connection_closed",
                            "connection_id_updated", "spin_bit_updated",   "connection_state_updated",
                            "path_assigned",          "mtu_updated"]

quic_event_list_         = ["version_information",     "alpn_information",    "parameters_set",
                            "parameters_restored",     "packet_sent",         "packet_received",
                            "packets_acked",            "datagrams_sent",      "datagrams_received",
                            "datagram_dropped",         "stream_state_updated", "frames_processed",
                            "stream_data_moved",        "datagram_data_moved",  "migration_state_updated",
                            "packet_dropped",            "packet_buffered"]

security_event_list_     = ["key_updated" ,   "key_discarded"]

recovery_event_list_     = ["rec_parameters_set", "rec_metrics_updated", "congestion_state_updated",
                            "loss_timer_updated",  "packet_lost", "marked_for_retransmit", 
                            "ecn_state_updated"]

http_event_list_         = ["http_parameters_set", "http_parameters_restored", "http_stream_type_set",
                            "http_frame_created",  "http_frame_parsed", "push_resolved", "http_setting_parsed"]

qpack_event_list_        = ["qpack_state_updated", "qpack_stream_state_updated", "dynamic_table_updated","headers_encoded",
                            "headers_decoded", "instruction_created", "instruction_parsed"]

packet_type_ = {0: "initial",  1: "0RTT", 2: "handshake" ,
                3: "retry", 4: "short_header", 5: "version_negotiation", 6: "unknown"}

packet_number_namespace_ = {0: "initial", 1: "handshake", 2: "application data", 3: "negotiation"}

packet_type_ = {0: "initial",  1: "0RTT", 2: "handshake" ,
                3: "retry", 4: "short_header", 5: "version_negotiation", 6: "unknown"}

packet_number_namespace_ = {0: "initial", 1: "handshake", 2: "application data", 3: "negotiation"}

send_stream_states_ = ["ready", "send", "data_sent", "reset_sent", "reset_received"]

recv_stream_states_ = ["receive", "size_known", "data_read", "reset_read", "reset_received", "reset_received"]

frame_type_   = ["PADDING", "PING", "ACK", "RESET_STREAM", "STOP_SENDING", "CRYPTO", "NEW_TOKEN",
                 "STREAM", "MAX_DATA", "MAX_STREAM_DATA", "MAX_STREAMS", "DATA_BLOCKED", "STREAM_DATA_BLOCKED", "STREAMS_BLOCKED",
                 "NEW_CONNECTION_ID", "RETIRE_CONNECTION_ID", "PATH_CHALLENGE", "PATH_RESPONSE", "CONNECTION_CLOSE", "HANDSHAKE_DONE",
                 "ACK_MP", "PATH_ABANDON", "PATH_STATUS", "DATAGRAM", "Extension"]

h3_stream_type_ = ["control", "push", "qpack_encode", "qpack_decode", "request", "bytestream", "unknown"]

h3_frame_type_ = ["data", "headers", "bidi_stream_type", "cancel_push", "settings", "push_promise", "goaway", "max_push_id", "unknown"]

last_scid_ = "initcid"

def get_path_id(line):
    segments = line.split('|')
    segments = [segment.strip() for segment in segments]
    assert(len(segments) > 1)
    for i in range(1,len(segments)):
        item = segments[i].split(':')
        item = [i.strip() for i in item]
        if(len(item) != 2):
            continue
        if item[0] == "path_id":
            return int(item[1])
    raise ValueError("no path_id")

def parse_line(line):
    event = {}
    event_scid = ""
    pattern = r'\[(.*?)\]'
    matches = re.findall(pattern, line)
    matches = [segment.strip() for segment in matches]
    if(len(matches) != 2):
        return None, event_scid
    time_str = matches[0]
    dt_object = datetime.datetime.strptime(time_str, "%Y/%m/%d %H:%M:%S %f")

    event["time"] = dt_object.timestamp() * 1000
    event["name"] = matches[1]

    if event["name"] in connectivity_event_list_:
        event["name"] = "connectivity:" + event["name"] 
    elif event["name"] in quic_event_list_:
        event["name"] = "quic:" + event["name"] 
    elif event["name"] in security_event_list_:
        event["name"] = "security:" + event["name"]
    elif event["name"] in recovery_event_list_:
        if event["name"] == "rec_parameters_set"  or  event["name"] == "rec_metrics_updated":
            event["name"] = event["name"][4:]
        event["name"] = "recovery:" + event["name"]
    elif event["name"] in http_event_list_:
        if event["name"] == "push_resolved":
            event["name"]  = "h3:push_resolved"
        else:
            event["name"] = "h3:" + event["name"][5:]
    else:
        return None, event_scid
    
    if matches[1] in ["packet_sent", "packet_received", "mtu_updated", "datagrams_sent", "datagrams_received", "packets_acked"]:
        event["path"] = get_path_id(line)
    
    if matches[1] == "packet_sent" or matches[1] == "packet_received":
        event["data"], event_scid = parse_packet_sent_and_recv(line)
        return event, event_scid
    
    if matches[1] == "datagrams_sent" or matches[1] == "datagrams_received":
        event["data"], event_scid = parse_datagrams_sent_or_recv(line)
        return event, event_scid
    
    if matches[1] in  ["server_listening", "connection_started", "connection_close", "connection_state_updated",
                        "path_assigned", "mtu_updated", "alpn_information","parameters_set", "packet_buffered", "packets_acked", "stream_state_updated",
                        "frames_processed", "stream_data_moved", "rec_parameters_set", "rec_metrics_updated", "congestion_state_updated", 
                        "packet_lost", "http_parameters_set", "http_frame_created", "http_frame_parse"]:
        function_name = "parse_" + matches[1]
        func = globals()[function_name]
        (event["data"], event_scid) = func(line)
        return event, event_scid
    else:
        return None, None



def endpoint_events_extraction(file_name, vantagepoint):
    assert(vantagepoint == "server" or vantagepoint == "client")
    conn_events = {
        "vantage_point" :  {"name": vantagepoint + "-view",
                             "type": vantagepoint
        },
        "title": "xquic qlog", 
        "description": "",
        "common_fields": { "ODCID": "", "time_format": "absolute" },
        "events": []
    }
    count = 0
    last_scid_ = "initcid"
    traces_log = {}
    with open(file_name, 'r',encoding='utf-8', errors='ignore') as file:
        for line in file:
            event, scid = parse_line(line)
            if (event is None):
                continue
            if (event is not None) and (scid != last_scid_):
                if count > 0:
                    if(scid in traces_log):
                        traces_log[scid]['events'] += conn_events['events']
                    else:
                        traces_log[last_scid_] = conn_events
                last_scid_ = scid
                conn_events = {
                    "title": "xquic-qlog json: " + vantagepoint, 
                    "description": "",
                    "common_fields": { "ODCID": scid, "time_format": "absolute" },
                    "vantage_point" :  {"name": vantagepoint + "-view",
                             "type": vantagepoint
                    },
                    "events": []
                }
                count = 1
                conn_events["events"].append(event)
            else:
                count += 1
                conn_events["events"].append(event)
    if(count > 1):
        scid = conn_events["common_fields"]["ODCID"]
        if(scid in traces_log):
            traces_log[scid]['events'] += conn_events['events']
        else:
            traces_log[scid] = conn_events
    return list(traces_log.values())


def parse_packet_sent_and_recv(line):
    data = {
            "header": {
              "packet_number": "unknown",
              "packet_type": "unknown"
            },
            "raw": {
              "length": 1280
            }
    }
    event_scid = "unknown"
    segments = line.split('|')
    segments = [segment.strip() for segment in segments]
    assert(len(segments) > 1)
    for i in range(1,len(segments)):
        item = segments[i].split(':')
        item = [i.strip() for i in item]
        if(len(item) != 2):
            continue
        if item[0] == "scid":
            event_scid = item[1]
        elif(item[0] == "pkt_type"):
            data["header"]["packet_type"] = item[1]
        elif(item[0] == "pkt_num"):
            data["header"]["packet_number"] = int(item[1])
        elif(item[0] == "size"):
            data["raw"]["length"] = int(item[1])
    return (data, event_scid)

def parse_server_listening(line):
    data = {
    }
    event_scid = "unknown"
    segments = line.split('|')
    segments = [segment.strip() for segment in segments]
    assert(len(segments) > 1)
    for i in range(1,len(segments)):
        item = segments[i].split(':')
        item = [i.strip() for i in item]
        if(len(item) != 2):
            continue
        if item[0] == "scid":
            event_scid = item[1]
        else:
            if item[0].startswith("port"):
                data[item[0]] = int(item[1])
            else:
                data[item[0]] = item[1]
    return (data, event_scid)

def parse_connection_started(line):
     # [2024/05/14 11:34:11 641605] [connection_started] |scid:b59e52a51185db48|xqc_engine_packet_process|local|src_ip:127.0.0.1|src_port:35148|
    data = {
            "src_ip": "127.0.0.1",
            "dst_ip": "127.0.0.1",
            "src_port": 0,
            "dst_port": 0
    }
    event_scid = "unknown"
    segments = line.split('|')
    segments = [segment.strip() for segment in segments]
    assert(len(segments) > 1)
    for i in range(1,len(segments)):
        item = segments[i].split(':')
        item = [i.strip() for i in item]
        if(len(item) != 2):
            continue
        if item[0] == "scid":
            event_scid = item[1]
        elif(item[0] == "src_ip"):
            data["src_ip"] = item[1]
        elif(item[0] == "dst_ip"):
            data["dst_ip"] = item[1]
        elif(item[0] == "src_port"):
            data["src_port"] = int(item[1])
        elif(item[0] == "dst_port"):
            data["dst_port"] = int(item[1])
    return (data, event_scid)

def parse_connection_closed(line):
    # [2024/05/14 11:34:26 672332] [connection_closed] |scid:007cc254f81be8e78d765a2e63339fc99a66320d|xqc_conn_destroy|err_code:0|
    data = {
            "connection_code": 0
    }
    event_scid = "unknown"
    segments = line.split('|')
    segments = [segment.strip() for segment in segments]
    assert(len(segments) > 1)
    for i in range(1,len(segments)):
        item = segments[i].split(':')
        item = [i.strip() for i in item]
        if(len(item) != 2):
            continue
        if item[0] == "scid":
            event_scid = item[1]
        elif item[0] == "err_code":
            data["connection_code"] = int(item[1])
            break
    return (data, event_scid)

def parse_connection_state_updated(line):
    data = {
            "new": "unknow"
    }
    event_scid = "unknown"
    segments = line.split('|')
    segments = [segment.strip() for segment in segments]
    assert(len(segments) > 1)
    for i in range(1,len(segments)):
        item = segments[i].split(':')
        item = [i.strip() for i in item]
        if(len(item) != 2):
            continue
        if item[0] == "scid":
            event_scid = item[1]
        elif item[0] == "new":
            data["new"] = item[1]
            break
    return (data, event_scid)

def parse_path_assigned(line):
    data = {
            "path_id": "unknow"
    }
    event_scid = "unknown"
    segments = line.split('|')
    segments = [segment.strip() for segment in segments]
    assert(len(segments) > 1)
    for i in range(1,len(segments)):
        item = segments[i].split(':')
        item = [i.strip() for i in item]
        if(len(item) != 2):
            continue
        if item[0] == "scid":
            event_scid = item[1]
        elif item[0] == "path_id":
            data["path_id"] = item[1]
            break
    return (data, event_scid)


def parse_mtu_updated(line):
    data = {
            "new": 0,
            "done": False
    }
    event_scid = "unknown"
    segments = line.split('|')
    segments = [segment.strip() for segment in segments]
    assert(len(segments) > 1)
    for i in range(1,len(segments)):
        item = segments[i].split(':')
        item = [i.strip() for i in item]
        if(len(item) != 2):
            continue
        if item[0] == "scid":
            event_scid = item[1]
        elif item[0] == "new":
            data["path_id"] = int(item[1])
        elif item[0] == "done":
            data["done"] = False
            if int(item[1]):
                data["done"] = True
    return (data, event_scid)

def parse_alpn_information(line):
    #[2024/05/14 11:34:11 548027] [alpn_information] 
    #|scid:007cc254f81be8e78d765a2e63339fc99a66320d|xqc_ssl_alpn_select_cb|client_alpn:h3 |server_alpn:h3 h3-29 h3-ext transport |selected_alpn:h3|
    data = {
            "server_alpns": [],
            "client_alpns": [],
            "chosen_alpn": {"string_value" : "unknown"}
    }
    event_scid = "unknown"
    segments = line.split('|')
    segments = [segment.strip() for segment in segments]
    assert(len(segments) > 1)
    for i in range(1,len(segments)):
        item = segments[i].split(':')
        item = [i.strip() for i in item]
        if(len(item) != 2):
            continue
        if item[0] == "scid":
            event_scid = item[1]
        elif item[0] == "client_alpn":
            alpns = item[1].split(' ')
            for alpn in alpns:
                temp = {"string_value" : alpn}
                data["client_alpns"].append(temp)
        elif item[0] == "server_alpn":
            alpns = item[1].split(' ')
            for alpn in alpns:
                temp = {"string_value" : alpn}
                data["server_alpns"].append(temp)
        elif item[0] == "selected_alpn":
            data["chosen_alpn"]["string_value"] = item[1]
    return (data, event_scid)


def parse_parameters_set(line):
    # [2024/05/14 11:34:11 547473] [tra_parameters_set] |scid:007cc254f81be8e78d765a2e63339fc99a66320d|
    # xqc_conn_create|local|migration:1|max_idle_timeout:120000|max_udp_payload_size:1500|active_connection_id_limit:8|max_data:0|
    data = {
            "max_idle_timeout": 0,
            "max_udp_payload_size": 0,
            "active_connection_id_limit": 0
    }
    event_scid = "unknown"
    segments = line.split('|')
    segments = [segment.strip() for segment in segments]
    assert(len(segments) > 1)
    for i in range(1,len(segments)):
        item = segments[i].split(':')
        item = [i.strip() for i in item]
        if(len(item) != 2):
            continue
        if item[0] == "scid":
            event_scid = item[1]
        elif item[0] in data.keys:
            data[item[0]] = int(item[1])
    return (data, event_scid)

def parse_packet_buffered(line):
    # [2024/05/14 11:34:11 693524] [packet_buffered] |scid:007cc254f81be8e78d765a2e63339fc99a66320d|
    # xqc_conn_buff_undecrypt_packet_in|pkt_pns:2|pkt_type:4|len:1216|
    data = {
            "header": {
              "packet_number": "unknown",
              "packet_type": "unknown"
            },
            "raw": {
              "length": 1280
            }
    }
    event_scid = "unknown"
    segments = line.split('|')
    segments = [segment.strip() for segment in segments]
    assert(len(segments) > 1)
    for i in range(1,len(segments)):
        item = segments[i].split(':')
        item = [i.strip() for i in item]
        if(len(item) != 2):
            continue
        if item[0] == "scid":
            event_scid = item[1]
        elif(item[0] == "pkt_type"):
            data["header"]["packet_type"] = packet_type_[int(item[1])]
        elif(item[0] == "pkt_num"):
            data["header"]["packet_number"] = int(item[1])
        elif(item[0] == "len"):
            data["raw"]["length"] = int(item[1])
    return (data, event_scid)

def parse_packets_acked(line):
    # [2024/05/14 11:34:11 642140] [packets_acked] |scid:007cc254f81be8e78d765a2e63339fc99a66320d|
    # xqc_process_ack_frame|pkt_space:0|high:0|low:0|path_id:0|
    data = {
            "packet_number_space": "unknown",
            "packet_numbers": []
    }
    event_scid = "unknown"
    low = 0
    high = -1
    segments = line.split('|')
    segments = [segment.strip() for segment in segments]
    assert(len(segments) > 1)
    for i in range(1,len(segments)):
        item = segments[i].split(':')
        item = [i.strip() for i in item]
        if(len(item) != 2):
            continue
        if item[0] == "scid":
            event_scid = item[1]
        elif(item[0] == "pkt_space"):
            data["packet_number_space"] = packet_number_namespace_[int(item[1])]
        elif(item[0] == "low"):
            low = int(item[1])
        elif(item[0] == "high"):
            high = int(item[1])
    if(low > high):
        return (None, event_scid)
    for i in range(low, high + 1):
        data["packet_numbers"].append(i)
    return (data, event_scid)

def parse_datagrams_sent_or_recv(line):
    # [2024/05/14 11:34:11 552331] [datagrams_sent] |scid:007cc254f81be8e78d765a2e63339fc99a66320d|xqc_send|size:1216|
    data = {
            "raw": {
              "length": 1280
            }
    }
    event_scid = "unknown"
    segments = line.split('|')
    segments = [segment.strip() for segment in segments]
    assert(len(segments) > 1)
    for i in range(1,len(segments)):
        item = segments[i].split(':')
        item = [i.strip() for i in item]
        if(len(item) != 2):
            continue
        if item[0] == "scid":
            event_scid = item[1]
        elif(item[0] == "size"):
            data["raw"]["length"] = int(item[1])
    return (data, event_scid)

def parse_stream_state_updated(line):
    # [2024/05/14 11:34:11 552613] [stream_state_updated] 
    # |scid:007cc254f81be8e78d765a2e63339fc99a66320d|xqc_stream_send_state_update|stream_id:3|send_stream|old:0|new:1|
    data = {
           "StreamType" : "bidirectional",
           "new": "unknown",
            "stream_side": "sending"
    }
    event_scid = "unknown"
    segments = line.split('|')
    segments = [segment.strip() for segment in segments]
    assert(len(segments) > 1)
    for i in range(1,len(segments)):
        item = segments[i].split(':')
        item = [i.strip() for i in item]
        if(len(item) == 1):
            if item[0] == "send_stream":
                data["stream_side"] = "sending"
            if item[0] == "recv_stream":
                data["stream_side"] = "receiving"
        if len(item) != 2:
            continue
        if item[0] == "scid":
            event_scid = item[1]
        if item[0] == "new":
            state = int(item[1])
            if data["stream_side"] == "sending":
                data["new"] = send_stream_states_[state]
            else:
                data["new"] = recv_stream_states_[state]
    return (data, event_scid)


def parse_frames_processed(line):
    data = {
           "frames":[{"frame_type" : "unknow"}]
    }
    event_scid = "unknown"
    segments = line.split('|')
    segments = [segment.strip() for segment in segments]
    assert(len(segments) > 1)
    frame_type = -1
    for i in range(1,len(segments)):
        item = segments[i].split(':')
        item = [i.strip() for i in item]
        if len(item) == 2 and item[0] == "scid":
            event_scid = item[1]
        if(len(item) == 2 and item[0] == "type"):
            frame_type = int(item[1])
            break
    data["frames"][0]["frame_type"] = frame_type_[frame_type].lower()
    type_str = frame_type_[frame_type]

    if type_str == "PADDING":
        for i in range(1,len(segments)):
            item = segments[i].split(':')
            item = [i.strip() for i in item]
            if(len(item) != 2):
                continue
            if item[0] == "length":
                data["frames"][0]["payload_length"] = int(item[1])
        return (data, event_scid)
    elif type_str == "PING":
        return (data, event_scid)
    
    elif type_str == "ACK":
        for i in range(1,len(segments)):
            item = segments[i].split(':')
            item = [i.strip() for i in item]
            if(len(item) != 2):
                continue
            if item[0] == "ack_range":
                s_clean = item[1].strip("{}")  
                pairs = s_clean.split(", ")  
                result = [list(map(int, pair.split(" - "))) for pair in pairs]
                for i in range(len(result)):
                    if result[i][0] == result[i][1]:
                        result[i] = [result[i][0]]
                data["frames"][0]["acked_ranges"] = result
        return (data, event_scid)
    
    elif type_str == "RESET_STREAM" or type_str == "STOP_SENDING":
        for i in range(1,len(segments)):
            item = segments[i].split(':')
            item = [i.strip() for i in item]
            if(len(item) != 2):
                continue
            if item[0] == "stream_id":
                data["frames"][0]["stream_id"] = int(item[1])
            elif item[0] == "err_code":
                data["frames"][0]["error_code"] = int(item[1])
            elif item[0] == "final_size":
                data["frames"][0]["final_size"] = int(item[1])
        return (data, event_scid)
    
    elif type_str == "CRYPTO":
        for i in range(1,len(segments)):
            item = segments[i].split(':')
            item = [i.strip() for i in item]
            if(len(item) != 2):
                continue
            if item[0] == "offset":
                data["frames"][0]["offset"] = int(item[1])
            elif item[0] == "length":
                data["frames"][0]["length"] = int(item[1])
        return (data, event_scid)
    
    elif type_str == "NEW_TOKEN":
        for i in range(1,len(segments)):
            item = segments[i].split(':')
            item = [i.strip() for i in item]
            if(len(item) != 2):
                continue
            if item[0] == "token":
                data["frames"][0]["token"] = item[1]
        return (data, event_scid)
    
    elif type_str == "STREAM":
        for i in range(1,len(segments)):
            item = segments[i].split(':')
            item = [i.strip() for i in item]
            if(len(item) != 2):
                continue
            if item[0] == "data_length":
                data["frames"][0]["length"] = int(item[1])
            elif item[0] == "data_offset":
                data["frames"][0]["offset"] = int(item[1])
            elif item[0] == "fin":
                data["frames"][0]["fin"] = False
                if int(item[1]):
                    data["frames"][0]["offset"] = True
        return (data, event_scid)
    
    elif type_str == "MAX_DATA":
        for i in range(1,len(segments)):
            item = segments[i].split(':')
            item = [i.strip() for i in item]
            if(len(item) != 2):
                continue
            if item[0] == "max_data":
                data["frames"][0]["maximum"] = int(item[1])
        return (data, event_scid)
    
    elif type_str == "MAX_STREAM_DATA":
        for i in range(1,len(segments)):
            item = segments[i].split(':')
            item = [i.strip() for i in item]
            if(len(item) != 2):
                continue
            if item[0] == "stream_id":
                data["frames"][0]["stream_id"] = int(item[1])
            elif item[0] == "max_stream_data":
                data["frames"][0]["maximum"] = int(item[1])
        return (data, event_scid)
    
    elif type_str == "MAX_STREAMS":
        for i in range(1,len(segments)):
            item = segments[i].split(':')
            item = [i.strip() for i in item]
            if(len(item) != 2):
                continue
            if item[0] == "stream_type":
                data["frames"][0]["stream_type"] = item[1]
            elif item[0] == "maximum":
                data["frames"][0]["maximum"] = int(item[1])
        return (data, event_scid)
    
    elif type_str == "DATA_BLOCKED":
        for i in range(1,len(segments)):
            item = segments[i].split(':')
            item = [i.strip() for i in item]
            if(len(item) != 2):
                continue
            if item[0] == "limit":
                data["frames"][0]["limit"] = int(item[1])
        return (data, event_scid)
    
    elif type_str == "STREAM_DATA_BLOCKED":
        for i in range(1,len(segments)):
            item = segments[i].split(':')
            item = [i.strip() for i in item]
            if(len(item) != 2):
                continue
            if item[0] == "stream_id":
                data["frames"][0]["stream_id"] = int(item[1])
            elif item[0] == "limit":
                data["frames"][0]["limit"] = int(item[1])
        return (data, event_scid)
    
    elif type_str == "STREAMS_BLOCKED":
        for i in range(1,len(segments)):
            item = segments[i].split(':')
            item = [i.strip() for i in item]
            if(len(item) != 2):
                continue
            if item[0] == "stream_type":
                data["frames"][0]["stream_type"] = item[1]
            elif item[0] == "limit":
                data["frames"][0]["limit"] = int(item[1])
        return (data, event_scid)
    
    elif type_str == "NEW_CONNECTION_ID":
        for i in range(1,len(segments)):
            item = segments[i].split(':')
            item = [i.strip() for i in item]
            if(len(item) != 2):
                continue
            if item[0] in ["sequence_number", "retire_prior_to", "connection_id_length"]:
                data["frames"][0][item[0]] = int(item[1])
            elif item[0] == "connection_id":
                data["frames"][0]["connection_id"] = item[1]
        return (data, event_scid)

    elif type_str == "CONNECTION_CLOSE":
        for i in range(1,len(segments)):
            item = segments[i].split(':')
            item = [i.strip() for i in item]
            if(len(item) != 2):
                continue
            if item[0] == "err_code":
                data["frames"][0]["error_code"] = int(item[1])
        return (data, event_scid)
    else:
        return (data, event_scid)

def parse_stream_data_moved(line):
    # [2024/05/14 11:34:11 547904] [stream_data_moved] |scid:007cc254f81be8e78d765a2e63339fc99a66320d|xqc_stream_send|ret:-610|stream_id:3|stream_send_offset:0|pkt_type:SHORT_HEADER|buff_1rtt:0|send_data_size:1|offset:0|fin:0|
    # stream_flag:17|conn:0000000002354F7C|conn_state:S_INIT|flag:TICKING TOKEN_OK UPPER_CONN_EXIST INIT_RECVD |from:application|to:transport|
    data = {
           "stream_id" : "unknown",
           "offset": 0,
           "length": 0,
           "from": "unknown",
           "to": "unknown"

    }
    event_scid = "unknown"
    segments = line.split('|')
    segments = [segment.strip() for segment in segments]
    assert(len(segments) > 1)
    for i in range(1,len(segments)):
        item = segments[i].split(':')
        item = [i.strip() for i in item]
        if(len(item) == 1):
            continue
        if item[0] == "stream_id":
            data["stream_id"] = int(item[1])
        elif item[0] == "stream_send_offset":
            data["offset"] = int(item[1])
        elif item[0] in ["from", "to"]:
            data[item[0]] = item[1]
        elif item[0] == "send_data_size":
            data["length"] = int(item[1])
        elif item[0] == "scid":
            event_scid = item[1]
        
    return (data, event_scid)


def parse_rec_parameters_set(line):
    data = {}
    event_scid = "unknown"
    segments = line.split('|')
    segments = [segment.strip() for segment in segments]
    assert(len(segments) > 1)
    for i in range(1,len(segments)):
        item = segments[i].split(':')
        item = [i.strip() for i in item]
        if(len(item) != 2):
            continue
        if item[0] == "scid":
            event_scid = item[1]
        else:
            data[item[0]] = int(item[1])
    return (data, event_scid)

def parse_rec_metrics_updated(line):
    # [2024/05/14 11:34:11 642348] [rec_metrics_updated] 
    # |scid:007cc254f81be8e78d765a2e63339fc99a66320d|xqc_send_ctl_on_ack_received|
    # cwnd:47152|inflight:1384|mode:0|applimit:0|pacing_rate:1477151|bw:13370|srtt:89748|latest_rtt:89748|ctl_rttvar:0|pto_count:89748|min_rtt:6|send:0|lost:0|tlp:29|recv:47152|
    data = {}
    event_scid = "unknown"
    segments = line.split('|')
    segments = [segment.strip() for segment in segments]
    assert(len(segments) > 1)
    for i in range(1,len(segments)):
        item = segments[i].split(':')
        item = [i.strip() for i in item]
        if(len(item) != 2):
            continue
        if item[0] == "scid":
            event_scid = item[1]
        elif item[0] == "cwnd":
            data["congestion_window"] = int(item[1])
        elif item[0] == "inflight":
            data["bytes_in_flight"] = int(item[1])
        elif item[0] == "pacing_rate":
            data["pacing_rate"] = int(item[1])
        elif item[0] == "pto_count":
            data["pto_count"] = int(item[1])
        elif item[0] == "ctl_rttvar":
            data["rtt_variance"] = int(item[1])
        elif item[0] == "min_rtt":
            data["min_rtt"] = int(item[1])
        elif item[0] == "latest_rtt":
            data["latest_rtt"] = int(item[1])

    return (data, event_scid)

def parse_congestion_state_updated(line):
    data = {"new" : "unknown"}
    event_scid = "unknown"
    segments = line.split('|')
    segments = [segment.strip() for segment in segments]
    assert(len(segments) > 1)
    for i in range(1,len(segments)):
        item = segments[i].split(':')
        item = [i.strip() for i in item]
        if(len(item) != 2):
            continue
        if item[0] == "scid":
            event_scid = item[1]
        elif item[0] == "new_state":
            data["new"] = item[1]
            return (data, event_scid)

def parse_packet_lost(line):
    data = {
            "header": {
              "packet_number": "unknown",
              "packet_type": "unknown"
            }
    }
    event_scid = "unknown"
    segments = line.split('|')
    segments = [segment.strip() for segment in segments]
    assert(len(segments) > 1)
    for i in range(1,len(segments)):
        item = segments[i].split(':')
        item = [i.strip() for i in item]
        if(len(item) != 2):
            continue
        if item[0] == "scid":
            event_scid = item[1]
        elif(item[0] == "pkt_type"):
            data["header"]["packet_type"] = packet_type_[int(item[1])]
        elif(item[0] == "pkt_num"):
            data["header"]["packet_number"] = int(item[1])
    return (data, event_scid)

def parse_http_parameters_set(line):
    data = {}
    event_scid = "unknown"
    segments = line.split('|')
    segments = [segment.strip() for segment in segments]
    assert(len(segments) > 1)
    for i in range(1,len(segments)):
        item = segments[i].split(':')
        item = [i.strip() for i in item]
        if(len(item) != 2):
            continue
        if item[0] == "scid":
            event_scid = item[1]
        elif(item[0] == "owner"):
            data["owner"] = item[1]
        else:
            data[item[0]] = int(item[1])
    return (data, event_scid)

def parse_http_stream_type_set(line):
    data = {"stream_id": "unknown",
             "stream_type": "unknown"
    }
    event_scid = "unknown"
    segments = line.split('|')
    segments = [segment.strip() for segment in segments]
    assert(len(segments) > 1)
    for i in range(1,len(segments)):
        item = segments[i].split(':')
        item = [i.strip() for i in item]
        if(len(item) != 2):
            continue
        if item[0] == "scid":
            event_scid = item[1]
        elif(item[0] == "stream_id"):
            data["stream_id"] = int(item[1])
        elif item[0] == "stream_type":
            data["stream_type"] = h3_stream_type_[int(item[1])]
    return (data, event_scid)

def parse_http_frame_created(line):
    data = {"stream_id": "unknown",
            "frame": {"frame_type": "unknown"}
    }
    event_scid = "unknown"
    segments = line.split('|')
    segments = [segment.strip() for segment in segments]
    assert(len(segments) > 1)
    h3_frame_type = "unknown"
    kv = {}
    for i in range(1,len(segments)):
        item = segments[i].split(':')
        item = [i.strip() for i in item]
        if(len(item) != 2):
            continue
        if item[0] == "scid":
            event_scid = item[1]
        if item[0] == "type":
            h3_frame_type = h3_frame_type_[int(item[1])]
        else:
            kv[item[0]] = item[1]
    data["frame"]["frame_type"] =  h3_frame_type
    data["stream_id"] =  kv["stream_id"]
    if h3_frame_type == "data":
        return (data, event_scid)
    elif h3_frame_type == "headers":
        header_list = []
        for i in range(1,len(segments)):
            if segments[i].startswith("{name"):
                match_header = re.findall(r'\{(.*?)\}', segments[i])
                temp_header = {"name": "unknown", "value": "/"}
                temp_header["name"] = match_header[0][5:]
                if len(match_header) == 2:
                    temp_header["value"] = match_header[0][6:]
                header_list.append(temp_header)
        data["frame"]["headers"] = header_list
        return (data, event_scid)

    elif h3_frame_type == "cancel_push" or h3_frame_type == "push_promise":
        data["frame"]["push_id"] = kv["push_id"]
        return (data, event_scid)

    elif h3_frame_type == "settings":
        temp_set = []
        for set_i in ["max_field_section_size", "max_pushes",
                        "qpack_max_table_capacity", "qpack_blocked_streams"]:
            temp_set.append({set_i: int(kv[set_i])})
        data["frame"]["settings"] = temp_set
        return (data, event_scid)
    else:
        return (data, event_scid)

def parse_http_frame_parsed(line):
    data = {}
    event_scid = "unknown"
    segments = line.split('|')
    segments = [segment.strip() for segment in segments]
    assert(len(segments) > 1)
    for i in range(1,len(segments)):
        item = segments[i].split(':')
        item = [i.strip() for i in item]
        if(len(item) != 2):
            continue
        if item[0] == "scid":
            event_scid = item[1]
        elif(item[0] == "stream_id"):
            data["stream_id"] = packet_type_[int(item[1])]
        elif(item[0] == "push_id"):
            data["push_id"] = int(item[1])
    return (data, event_scid)
    

def main():
    global last_scid_
    parser = argparse.ArgumentParser()
    parser.add_argument("--clog", help="xquic client log file")
    parser.add_argument("--slog", help="xquic server log file")
    parser.add_argument("--qlog_path", help="output json file, endswith .json", default="demo_qlog.json")
    args = parser.parse_args()
    if (args.clog is None) and (args.slog is None):
        print("Usage: must provide either --clog or --slog argument")
        sys.exit(1)
    if (args.clog is not None) and (not os.path.isfile(args.clog)):
        print(f"Error: The log '{args.clog}' does not exist.")
        sys.exit(1)
    if (args.slog is not None) and (not os.path.isfile(args.slog)):
        print(f"Error: The log '{args.slog}' does not exist.")
        sys.exit(1)
    if (args.qlog_path is not None) and (not args.qlog_path.endswith(".json")):
        print(f"Error: The qlog_path should endswith .json.")
        sys.exit(1)

    data = { "qlog_version": "0.4",
        "qlog_format": "JSON",
        "title": "xquic qlog",
        "description": "this is a demo qlog json of qlog-draft-07",
        "traces": [] 
    }
    if(args.slog is not None):
        server_traces = endpoint_events_extraction(args.slog, "server")
        data["traces"] += server_traces

    if(args.clog is not None):
        client_traces = endpoint_events_extraction(args.clog, "client")
        data["traces"] += client_traces

    json_output = json.dumps(data, indent=4)
    
    with open(args.qlog_path, 'w') as out_file:
        out_file.write(json_output)


if __name__ == "__main__":
    main()