def get_parser()

in assets/training/distillation/src/generate_data_preprocess.py [0:0]

• 107 lines of code
• 1 McCabe index (conditional complexity)

def get_parser():
    """
    Add arguments and returns the parser. Here we add all the arguments for all the tasks.

    Those arguments that are not relevant for the input task should be ignored.
    """
    parser = argparse.ArgumentParser(
        description="Model selector for hugging face models", allow_abbrev=False
    )

    # File I/O
    parser.add_argument(
        "--train_file_path",
        type=str,
        help="Input train file path",
    )

    parser.add_argument(
        "--validation_file_path",
        default=None,
        type=str,
        help="Input validation file path",
    )

    parser.add_argument(
        "--generated_train_file_path",
        type=Path,
        default=None,
        help="file to save the generated training data",
    )

    parser.add_argument(
        "--generated_validation_file_path",
        type=Path,
        default=None,
        help="file to save the generated validation data",
    )

    # add optional data-generator params
    parser.add_argument(
        "--teacher_model_endpoint_name",
        type=str,
        required=False,
        help="Teacher model endpoint name",
    )
    parser.add_argument(
        "--teacher_model_endpoint_key",
        type=str,
        required=False,
        help="Teacher model endpoint key",
    )

    parser.add_argument(
        "--teacher_model_endpoint_url",
        type=str,
        required=True,
        help="Teacher model endpoint URL",
    )

    parser.add_argument(
        "--teacher_model_max_new_tokens",
        type=int,
        required=False,
        default=DEFAULT_MAX_NEW_TOKENS,
        help="Teacher model max_tokens parameter",
    )
    parser.add_argument(
        "--teacher_model_temperature",
        type=float,
        required=False,
        default=DEFAULT_TEMPERATURE,
        help="Teacher model temperature parameter",
    )
    parser.add_argument(
        "--teacher_model_top_p",
        type=float,
        required=False,
        default=DEFAULT_TOP_P,
        help="Teacher model top-p parameter",
    )
    parser.add_argument(
        "--teacher_model_frequency_penalty",
        type=float,
        required=False,
        help="Teacher model frequency parameter",
    )
    parser.add_argument(
        "--teacher_model_presence_penalty",
        type=float,
        required=False,
        help="Teacher model presense penalty",
    )
    parser.add_argument(
        "--teacher_model_stop", type=str, required=False, help="Teacher model stop "
    )

    parser.add_argument(
        "--enable_chain_of_thought",
        type=str,
        required=False,
        default="false",
        help="This enables Chain of Thought",
    )

    parser.add_argument(
        "--enable_chain_of_density",
        type=str,
        required=False,
        default="false",
        help="This enables Chain of Density for Summarization",
    )

    parser.add_argument(
        "--max_len_summary",
        type=int,
        required=False,
        default=DEFAULT_MAX_LEN_SUMMARY,
        help="Maximum word count for text summarization ",
    )

    parser.add_argument(
        "--data_generation_task_type",
        type=str,
        required=True,
        help="""Data generation task type. Supported values are: