in codes/rnn_training/train_nli_w2v.py [0:0]
def HyperEvaluate(config):
print(config)
parser = argparse.ArgumentParser()
parser.add_argument("--node-ip-address=") # ,192.168.2.19
parser.add_argument("--node-manager-port=")
parser.add_argument("--object-store-name=")
parser.add_argument(
"--raylet-name="
) # /tmp/ray/session_2020-07-15_12-00-45_292745_38156/sockets/raylet
parser.add_argument("--redis-address=") # 192.168.2.19:6379
parser.add_argument("--config-list=", action="store_true") #
parser.add_argument("--temp-dir=") # /tmp/ray
parser.add_argument("--redis-password=") # 5241590000000000
# /////////NLI-Args//////////////
parser = argparse.ArgumentParser(description="NLI training")
# paths
parser.add_argument(
"--nlipath",
type=str,
default=config["dataset"],
help="NLI data (SNLI or MultiNLI)",
)
parser.add_argument(
"--outputdir", type=str, default="savedir_van/", help="Output directory"
)
parser.add_argument("--outputmodelname", type=str, default="model.pickle")
parser.add_argument(
"--word_emb_path",
type=str,
default="dataset/GloVe/glove.840B.300d.txt",
help="word embedding file path",
)
# training
parser.add_argument("--n_epochs", type=int, default=20)
parser.add_argument("--batch_size", type=int, default=128)
parser.add_argument(
"--dpout_model", type=float, default=0.2, help="encoder dropout"
)
parser.add_argument(
"--dpout_fc", type=float, default=0.2, help="classifier dropout"
)
parser.add_argument(
"--nonlinear_fc", type=float, default=0, help="use nonlinearity in fc"
)
parser.add_argument(
"--optimizer", type=str, default="adam,lr=0.001", help="adam or sgd,lr=0.1"
)
parser.add_argument(
"--lrshrink", type=float, default=5, help="shrink factor for sgd"
)
parser.add_argument("--decay", type=float, default=0.99, help="lr decay")
parser.add_argument("--minlr", type=float, default=1e-10, help="minimum lr")
parser.add_argument(
"--max_norm", type=float, default=5.0, help="max norm (grad clipping)"
)
# model
parser.add_argument(
"--encoder_type",
type=str,
default=config["encoder_type"],
help="see list of encoders",
)
parser.add_argument(
"--enc_lstm_dim", type=int, default=200, help="encoder nhid dimension"
) # 2048
parser.add_argument(
"--n_enc_layers", type=int, default=1, help="encoder num layers"
)
parser.add_argument("--fc_dim", type=int, default=200, help="nhid of fc layers")
parser.add_argument(
"--n_classes",
type=int,
default=config["num_classes"],
help="entailment/neutral/contradiction",
)
parser.add_argument("--pool_type", type=str, default="max", help="max or mean")
# gpu
parser.add_argument("--gpu_id", type=int, default=3, help="GPU ID")
parser.add_argument("--seed", type=int, default=config["seed"], help="seed")
# data
parser.add_argument(
"--word_emb_dim", type=int, default=300, help="word embedding dimension"
)
parser.add_argument(
"--word_emb_type",
type=str,
default="normal",
help="word embedding type, either glove or normal",
)
# comet
# parser.add_argument("--comet_apikey", type=str, default='', help="comet api key")
# parser.add_argument("--comet_workspace", type=str, default='', help="comet workspace")
# parser.add_argument("--comet_project", type=str, default='', help="comet project name")
# parser.add_argument("--comet_disabled", action='store_true', help="if true, disable comet")
params, _ = parser.parse_known_args()
print("Came here")
exp_folder = os.path.join(
params.outputdir,
params.nlipath,
params.encoder_type,
"exp_seed_{}".format(params.seed),
)
if not os.path.exists(exp_folder):
os.makedirs(exp_folder)
# set proper name
save_folder_name = os.path.join(exp_folder, "model")
if not os.path.exists(save_folder_name):
os.makedirs(save_folder_name)
test_sample_folder = os.path.join(exp_folder, "samples_test")
if not os.path.exists(test_sample_folder):
os.makedirs(test_sample_folder)
params.outputmodelname = os.path.join(
save_folder_name, "{}_model.pkl".format(params.encoder_type)
)
# print parameters passed, and all parameters
print("\ntogrep : {0}\n".format(sys.argv[1:]))
print(params)
pr = vars(params)
# ex = OfflineExperiment(
# workspace=pr['comet_workspace'],
# project_name=pr['comet_project'],
# disabled=pr['comet_disabled'],
# offline_directory= os.path.join(save_folder_name,'comet_runs'))
#
# ex.log_parameters(pr)
# ex.set_name(pr['encoder_type'])
"""
SEED
"""
# np.random.seed(params.seed)
# torch.manual_seed(params.seed)
device = "cpu"
if torch.cuda.is_available():
device = "cuda"
# torch.cuda.manual_seed(params.seed)
"""
DATA
"""
train, valid, test, vocab, label_vocab = DataIteratorGlove(
batch_size=params.batch_size,
dataset=params.nlipath,
max_length=20,
prefix="processed_",
)
# word_vec = build_vocab(train['s1'] + train['s2'] +
# valid['s1'] + valid['s2'] +
# test['s1'] + test['s2'], params.word_emb_path, emb_dim=params.word_emb_dim,
# wtype=params.word_emb_type)
# for split in ['s1', 's2']:
# for data_type in ['train', 'valid', 'test']:
# eval(data_type)[split] = np.array([['<s>'] +
# [word for word in sent.split() if word in word_vec] +
# ['</s>'] for sent in eval(data_type)[split]])
# Train label class balancing
weights = [2, 2, 2, 0.3, 7, 2, 6]
# invert the weights by values
word_vec = getEmbeddingWeights(vocab.itos, params.nlipath)
print("Embeddings loaded")
"""
MODEL
"""
# model config
config_nli_model = {
"n_words": len(vocab),
"word_emb_dim": params.word_emb_dim,
"enc_lstm_dim": params.enc_lstm_dim,
"n_enc_layers": params.n_enc_layers,
"dpout_model": params.dpout_model,
"dpout_fc": params.dpout_fc,
"fc_dim": params.fc_dim,
"bsize": params.batch_size,
"n_classes": params.n_classes,
"pool_type": params.pool_type,
"nonlinear_fc": params.nonlinear_fc,
"encoder_type": params.encoder_type,
"use_cuda": True,
}
# model
encoder_types = [
"InferSent",
"BLSTMprojEncoder",
"BGRUlastEncoder",
"InnerAttentionMILAEncoder",
"InnerAttentionYANGEncoder",
"InnerAttentionNAACLEncoder",
"ConvNetEncoder",
"LSTMEncoder",
]
assert params.encoder_type in encoder_types, "encoder_type must be in " + str(
encoder_types
)
nli_net = NLINet(config_nli_model, weights=word_vec)
print(nli_net)
weight = torch.FloatTensor(weights)
loss_fn = nn.CrossEntropyLoss()
loss_fn.size_average = False
# optimizer
optim_fn, optim_params = get_optimizer(params.optimizer)
optimizer = optim_fn(nli_net.parameters(), **optim_params)
# scheduler = StepLR(optimizer, step_size=5, gamma=0.5)
# cuda by default
nli_net.to(device)
loss_fn.to(device)
"""
TRAIN
"""
train_config = {
"val_acc_best": -1e10,
"adam_stop": False,
"stop_training": False,
"lr": optim_params["lr"] if "sgd" in params.optimizer else None,
}
"""
Train model on Natural Language Inference task
"""
epoch = 0
while not train_config["stop_training"] and epoch <= params.n_epochs:
epoch += 1
train_acc = trainepoch(nli_net, train, optimizer, loss_fn, epoch, params)
eval_acc, optimizer, train_config = evaluate(
nli_net, valid, optimizer, epoch, train_config, params, eval_type="valid"
)
# scheduler.step()
lock = FileLock(os.path.join(save_folder_name, "logs.txt" + ".new.lock"))
with lock:
with open(os.path.join(save_folder_name, "logs.txt"), "a") as f:
f.write(
f"| Epoch: {epoch:03} | Train Acc: {train_acc:.3f} | Val. Acc: {eval_acc:.3f} \n"
)
lock.release()
# Run best model on test set.
# nli_net.load_state_dict(torch.load(params.outputmodelname))
#
# print('\nTEST : Epoch {0}'.format(epoch))
# valid_acc, _, _ = evaluate(nli_net, valid, optimizer, epoch, train_config, params, eval_type='valid', test_folder = None, inv_label = label_vocab.itos, itos_vocab = vocab.itos, final_eval=True)
# test_acc, _, _ = evaluate(nli_net, test, optimizer, epoch, train_config, params, eval_type='test', test_folder = test_sample_folder, inv_label = label_vocab.itos, itos_vocab = vocab.itos, final_eval=True)
# lock = FileLock(os.path.join(save_folder_name,'logs.txt'+'.new.lock'))
# with lock:
# with open(os.path.join(save_folder_name,'logs.txt'),'a') as f:
# f.write(f'| Epoch: {epoch:03} | Test Acc: {test_acc:.3f} | Val. Acc: {valid_acc:.3f} \n')
# lock.release()
# ex.log_asset(file_name=res_file, file_like_object=open(res_file, 'r'))
# Save encoder instead of full model
# torch.save(nli_net.encoder.state_dict(), params.outputmodelname)
# Save word vectors
return eval_acc