def generate_filled_missing_by_field()

in decisionai_plugin/common/util/data.py [0:0]

• 43 lines of code
• 13 McCabe index (conditional complexity)

def generate_filled_missing_by_field(series_data, start, end, gran, custom_in_seconds, fill_type:Fill, fill_value, fields=None):
    if fields is None:
        fields = []
    fields.insert(0, VALUE)

    data_panel = {}

    full_data_range = pd.date_range(start=start, end=get_time_offset(end, (gran, custom_in_seconds), -1), freq=convert_freq(Gran[gran], custom_in_seconds))
    full_data_range = pd.DataFrame(full_data_range, columns=[TIMESTAMP])
    full_data_range[TIMESTAMP] = full_data_range[TIMESTAMP].dt.tz_localize(None)
    # set index to improve merging performance
    full_data_range = full_data_range.set_index(TIMESTAMP)

    series_info = []
    for series in series_data:
        if series is None:
            pass

        metric_id = series.metric_id
        series_id = series.series_id
        taglist = series.dim
        series_info.append([metric_id, series_id, taglist])
        
        ts_df = pd.DataFrame([value[TIMESTAMP] for value in series.value], columns=[TIMESTAMP])
        ts_df[TIMESTAMP] = ts_df[TIMESTAMP].dt.tz_localize(None)
        ts_df.set_index(TIMESTAMP, inplace=True)

        for field in fields:
            ts_df[series_id] = [value[field] for value in series.value]
            ts_df = pd.merge(full_data_range, ts_df, how='left', on=TIMESTAMP)

            if field not in data_panel:
                data_panel[field] = ts_df
            else:
                data_panel[field][series_id] = ts_df[series_id]

    timestamps = []
    cols_len = len(data_panel[VALUE].columns.values.tolist())
    for index, row in data_panel[VALUE].iterrows():
        not_missing = False
        for i in range(cols_len):
            if pd.notna(row[i]):
                not_missing = True
                break
        timestamps.append([index, not_missing])

    for field in fields:
        if field == VALUE:
            data_panel[field] = fill_missing(data_panel[field], fill_type, fill_value)
        else:
            data_panel[field] = data_panel[field].fillna(0)
        data_panel[field].reset_index(drop=False, inplace=True)
    
    return data_panel, series_info, timestamps