crop_yield_prediction/train_c3d.py [18:129]: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - def prep_data(batch_X, batch_y, cuda): batch_X, batch_y = Variable(batch_X), Variable(batch_y) if cuda: batch_X, batch_y = batch_X.cuda(), batch_y.cuda() return batch_X, batch_y def train_epoch(model, train_dataloader, optimizer, cuda): ''' Epoch operation in training phase''' model.train() if cuda: model.cuda() n_batches = len(train_dataloader) sum_loss = 0 for batch_X, batch_y in train_dataloader: batch_X, batch_y = prep_data(batch_X, batch_y, cuda) # forward optimizer.zero_grad() pred = model(batch_X) loss_func = torch.nn.MSELoss() loss = loss_func(pred, batch_y) loss.backward() optimizer.step() # note keeping sum_loss += loss.item() avg_loss = sum_loss / n_batches return avg_loss def cal_performance(prediction, y): rmse = np.around(sqrt(mean_squared_error(y, prediction)), 3) r2 = np.around(r2_score(y, prediction), 3) corr = tuple(map(lambda x: np.around(x, 3), pearsonr(y, prediction)))[0] return rmse, r2, corr def eval_epoch(model, validation_dataloader, cuda): ''' Epoch operation in evaluation phase ''' model.eval() if cuda: model.cuda() n_batches = len(validation_dataloader) n_samples = len(validation_dataloader.dataset) batch_size = validation_dataloader.batch_size predictions = torch.zeros(n_samples) # collect y as batch_y has been shuffled y = torch.zeros(n_samples) sum_loss = 0 with torch.no_grad(): for i, (batch_X, batch_y) in enumerate(validation_dataloader): batch_X, batch_y = prep_data(batch_X, batch_y, cuda) # forward pred = model(batch_X) loss_func = torch.nn.MSELoss() loss = loss_func(pred, batch_y) start = i * batch_size end = start + batch_size if i != n_batches - 1 else n_samples predictions[start:end] = pred y[start:end] = batch_y sum_loss += loss.item() if cuda: predictions, y = predictions.cpu(), y.cpu() predictions, y = predictions.data.numpy(), y.data.numpy() rmse, r2, corr = cal_performance(predictions, y) avg_loss = sum_loss / n_batches return avg_loss, rmse, r2, corr def eval_test(X_dir, X_test_indices, y_test, n_tsteps, max_index, n_triplets_per_file, batch_size, model_dir, model, epochs, year, exp_idx, log_file): with open(log_file, 'a') as f: print('Predict year {}'.format(year), file=f, flush=True) print('Test size {}'.format(y_test.shape[0]), file=f, flush=True) print('Experiment {}'.format(exp_idx), file=f, flush=True) cuda = torch.cuda.is_available() models = [] for epoch_i in range(epochs): models.append('{}/{}_{}_epoch{}.tar'.format(model_dir, exp_idx, year, epoch_i)) best_model = '{}/{}_{}_best.tar'.format(model_dir, exp_idx, year) models.append(best_model) for model_file in models: checkpoint = torch.load(model_file) if cuda else torch.load(model_file, map_location='cpu') model.load_state_dict(checkpoint['model_state_dict']) model.eval() if cuda: model.cuda() - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - crop_yield_prediction/train_cnn_lstm.py [18:129]: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - def prep_data(batch_X, batch_y, cuda): batch_X, batch_y = Variable(batch_X), Variable(batch_y) if cuda: batch_X, batch_y = batch_X.cuda(), batch_y.cuda() return batch_X, batch_y def train_epoch(model, train_dataloader, optimizer, cuda): ''' Epoch operation in training phase''' model.train() if cuda: model.cuda() n_batches = len(train_dataloader) sum_loss = 0 for batch_X, batch_y in train_dataloader: batch_X, batch_y = prep_data(batch_X, batch_y, cuda) # forward optimizer.zero_grad() pred = model(batch_X) loss_func = torch.nn.MSELoss() loss = loss_func(pred, batch_y) loss.backward() optimizer.step() # note keeping sum_loss += loss.item() avg_loss = sum_loss / n_batches return avg_loss def cal_performance(prediction, y): rmse = np.around(sqrt(mean_squared_error(y, prediction)), 3) r2 = np.around(r2_score(y, prediction), 3) corr = tuple(map(lambda x: np.around(x, 3), pearsonr(y, prediction)))[0] return rmse, r2, corr def eval_epoch(model, validation_dataloader, cuda): ''' Epoch operation in evaluation phase ''' model.eval() if cuda: model.cuda() n_batches = len(validation_dataloader) n_samples = len(validation_dataloader.dataset) batch_size = validation_dataloader.batch_size predictions = torch.zeros(n_samples) # collect y as batch_y has been shuffled y = torch.zeros(n_samples) sum_loss = 0 with torch.no_grad(): for i, (batch_X, batch_y) in enumerate(validation_dataloader): batch_X, batch_y = prep_data(batch_X, batch_y, cuda) # forward pred = model(batch_X) loss_func = torch.nn.MSELoss() loss = loss_func(pred, batch_y) start = i * batch_size end = start + batch_size if i != n_batches - 1 else n_samples predictions[start:end] = pred y[start:end] = batch_y sum_loss += loss.item() if cuda: predictions, y = predictions.cpu(), y.cpu() predictions, y = predictions.data.numpy(), y.data.numpy() rmse, r2, corr = cal_performance(predictions, y) avg_loss = sum_loss / n_batches return avg_loss, rmse, r2, corr def eval_test(X_dir, X_test_indices, y_test, n_tsteps, max_index, n_triplets_per_file, batch_size, model_dir, model, epochs, year, exp_idx, log_file): with open(log_file, 'a') as f: print('Predict year {}'.format(year), file=f, flush=True) print('Test size {}'.format(y_test.shape[0]), file=f, flush=True) print('Experiment {}'.format(exp_idx), file=f, flush=True) cuda = torch.cuda.is_available() models = [] for epoch_i in range(epochs): models.append('{}/{}_{}_epoch{}.tar'.format(model_dir, exp_idx, year, epoch_i)) best_model = '{}/{}_{}_best.tar'.format(model_dir, exp_idx, year) models.append(best_model) for model_file in models: checkpoint = torch.load(model_file) if cuda else torch.load(model_file, map_location='cpu') model.load_state_dict(checkpoint['model_state_dict']) model.eval() if cuda: model.cuda() - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -