def main()

in phi3/dataset-preparation/combine_tokenizer.py [0:0]

• 7 lines of code
• 1 McCabe index (conditional complexity)

def main():
    parser = argparse.ArgumentParser()

    # Dataset Arguments
    parser.add_argument("--tokenizer1", type=str, default="tokenizer-minstral-orig-txt", help="")
    parser.add_argument("--tokenizer2", type=str, default="tokenizer-ko-nsmc-txt", help="")
    parser.add_argument("--save_dir", type=str, default="tokenizer-new-1000", help="")

    args = parser.parse_args()
    combine_tokenizers(args)