crop_yield_train_cnn_lstm.py [52:83]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    dim_y = pd.read_csv('{}/dim_y.csv'.format(data_dir))
    dim_y = dim_y.astype({'state': int, 'county': int, 'year': int, 'value': float, 'lat': float, 'lon': float})
    max_index = len(dim_y) - 1

    results = dict()
    for year in range(start_year, end_year + 1):
        print('Predict year {}......'.format(year))

        test_idx = (dim_y['year'] == year)
        valid_idx = (dim_y['year'] == (year - 1))
        if train_years is None:
            train_idx = (dim_y['year'] < (year - 1))
        else:
            train_idx = (dim_y['year'] < (year - 1)) & (dim_y['year'] >= (year - 1 - train_years))

        y_valid, y_train = np.array(dim_y.loc[valid_idx]['value']), np.array(dim_y.loc[train_idx]['value'])
        y_test, dim_test = np.array(dim_y.loc[test_idx]['value']), np.array(dim_y.loc[test_idx][['state', 'county']])

        test_indices = [i for i, x in enumerate(test_idx) if x]
        valid_indices = [i for i, x in enumerate(valid_idx) if x]
        train_indices = [i for i, x in enumerate(train_idx) if x]

        # check if the indices are sequential
        assert all(elem == 1 for elem in [y - x for x, y in zip(test_indices[:-1], test_indices[1:])])
        assert all(elem == 1 for elem in [y - x for x, y in zip(valid_indices[:-1], valid_indices[1:])])
        assert all(elem == 1 for elem in [y - x for x, y in zip(train_indices[:-1], train_indices[1:])])
        print('Train size {}, valid size {}, test size {}'.format(y_train.shape[0], y_valid.shape[0], y_test.shape[0]))

        test_corr_lis, test_r2_lis, test_rmse_lis = [], [], []
        test_prediction_lis = []
        for i in range(n_experiment):
            print('Experiment {}'.format(i))
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



crop_yield_train_semi_transformer.py [82:113]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    dim_y = pd.read_csv('{}/dim_y.csv'.format(data_dir))
    dim_y = dim_y.astype({'state': int, 'county': int, 'year': int, 'value': float, 'lat': float, 'lon': float})
    max_index = len(dim_y) - 1

    results = dict()
    for year in range(start_year, end_year + 1):
        print('Predict year {}......'.format(year))

        test_idx = (dim_y['year'] == year)
        valid_idx = (dim_y['year'] == (year - 1))
        if train_years is None:
            train_idx = (dim_y['year'] < (year - 1))
        else:
            train_idx = (dim_y['year'] < (year - 1)) & (dim_y['year'] >= (year - 1 - train_years))

        y_valid, y_train = np.array(dim_y.loc[valid_idx]['value']), np.array(dim_y.loc[train_idx]['value'])
        y_test, dim_test = np.array(dim_y.loc[test_idx]['value']), np.array(dim_y.loc[test_idx][['state', 'county']])

        test_indices = [i for i, x in enumerate(test_idx) if x]
        valid_indices = [i for i, x in enumerate(valid_idx) if x]
        train_indices = [i for i, x in enumerate(train_idx) if x]

        # check if the indices are sequential
        assert all(elem == 1 for elem in [y - x for x, y in zip(test_indices[:-1], test_indices[1:])])
        assert all(elem == 1 for elem in [y - x for x, y in zip(valid_indices[:-1], valid_indices[1:])])
        assert all(elem == 1 for elem in [y - x for x, y in zip(train_indices[:-1], train_indices[1:])])
        print('Train size {}, valid size {}, test size {}'.format(y_train.shape[0], y_valid.shape[0], y_test.shape[0]))

        test_corr_lis, test_r2_lis, test_rmse_lis = [], [], []
        test_prediction_lis = []
        for i in range(n_experiment):
            print('Experiment {}'.format(i))
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -