distilbertqatrain.py [41:117]: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - def read_squad(path): # open JSON file and load intro dictionary with open(path, 'rb') as f: squad_dict = json.load(f) # initialize lists for contexts, questions, and answers contexts = [] questions = [] answers = [] # iterate through all data in squad data for group in squad_dict['data']: for passage in group['paragraphs']: context = passage['context'] for qa in passage['qas']: question = qa['question'] # check if we need to be extracting from 'answers' or 'plausible_answers' if 'plausible_answers' in qa.keys(): access = 'plausible_answers' else: access = 'answers' for answer in qa[access]: # append data to lists contexts.append(context) questions.append(question) answers.append(answer) # return formatted data lists return contexts, questions, answers def add_end_idx(answers, contexts): # loop through each answer-context pair for answer, context in zip(answers, contexts): # gold_text refers to the answer we are expecting to find in context gold_text = answer['text'] # we already know the start index start_idx = answer['answer_start'] # and ideally this would be the end index... end_idx = start_idx + len(gold_text) # ...however, sometimes squad answers are off by a character or two if context[start_idx:end_idx] == gold_text: # if the answer is not off :) answer['answer_end'] = end_idx else: # this means the answer is off by 1-2 tokens for n in [1, 2]: if context[start_idx-n:end_idx-n] == gold_text: answer['answer_start'] = start_idx - n answer['answer_end'] = end_idx - n # convert our character start/end positions to token start/end positions def add_token_positions(encodings, answers): start_positions = [] end_positions = [] for i in range(len(answers)): start_positions.append(encodings.char_to_token(i, answers[i]['answer_start'])) end_positions.append(encodings.char_to_token(i, answers[i]['answer_end'] - 1)) # if start position is None, the answer passage has been truncated if start_positions[-1] is None: start_positions[-1] = tokenizer.model_max_length if end_positions[-1] is None: end_positions[-1] = tokenizer.model_max_length encodings.update({'start_positions': start_positions, 'end_positions': end_positions}) if __name__ == "__main__": parser = argparse.ArgumentParser() # hyperparameters sent by the client are passed as command-line arguments to the script. parser.add_argument("--epochs", type=int, default=3) parser.add_argument("--train_batch_size", type=int, default=32) parser.add_argument("--eval_batch_size", type=int, default=64) parser.add_argument("--warmup_steps", type=int, default=500) parser.add_argument("--model_name", type=str,default='distilbert-base-uncased-distilled-squad') parser.add_argument("--learning_rate", type=str, default=5e-5) - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - scripts/train.py [27:103]: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - def read_squad(path): # open JSON file and load intro dictionary with open(path, 'rb') as f: squad_dict = json.load(f) # initialize lists for contexts, questions, and answers contexts = [] questions = [] answers = [] # iterate through all data in squad data for group in squad_dict['data']: for passage in group['paragraphs']: context = passage['context'] for qa in passage['qas']: question = qa['question'] # check if we need to be extracting from 'answers' or 'plausible_answers' if 'plausible_answers' in qa.keys(): access = 'plausible_answers' else: access = 'answers' for answer in qa[access]: # append data to lists contexts.append(context) questions.append(question) answers.append(answer) # return formatted data lists return contexts, questions, answers def add_end_idx(answers, contexts): # loop through each answer-context pair for answer, context in zip(answers, contexts): # gold_text refers to the answer we are expecting to find in context gold_text = answer['text'] # we already know the start index start_idx = answer['answer_start'] # and ideally this would be the end index... end_idx = start_idx + len(gold_text) # ...however, sometimes squad answers are off by a character or two if context[start_idx:end_idx] == gold_text: # if the answer is not off :) answer['answer_end'] = end_idx else: # this means the answer is off by 1-2 tokens for n in [1, 2]: if context[start_idx-n:end_idx-n] == gold_text: answer['answer_start'] = start_idx - n answer['answer_end'] = end_idx - n # convert our character start/end positions to token start/end positions def add_token_positions(encodings, answers): start_positions = [] end_positions = [] for i in range(len(answers)): start_positions.append(encodings.char_to_token(i, answers[i]['answer_start'])) end_positions.append(encodings.char_to_token(i, answers[i]['answer_end'] - 1)) # if start position is None, the answer passage has been truncated if start_positions[-1] is None: start_positions[-1] = tokenizer.model_max_length if end_positions[-1] is None: end_positions[-1] = tokenizer.model_max_length encodings.update({'start_positions': start_positions, 'end_positions': end_positions}) if __name__ == "__main__": parser = argparse.ArgumentParser() # hyperparameters sent by the client are passed as command-line arguments to the script. parser.add_argument("--epochs", type=int, default=3) parser.add_argument("--train_batch_size", type=int, default=32) parser.add_argument("--eval_batch_size", type=int, default=64) parser.add_argument("--warmup_steps", type=int, default=500) parser.add_argument("--model_name", type=str,default='distilbert-base-uncased-distilled-squad') parser.add_argument("--learning_rate", type=str, default=5e-5) - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -