def get_parser()

in assets/training/distillation/src/generate_data.py [0:0]
128 lines of code
1 McCabe index (conditional complexity)

def get_parser():
    """
    Add arguments and returns the parser. Here we add all the arguments for all the tasks.

    Those arguments that are not relevant for the input task should be ignored.
    """
    parser = argparse.ArgumentParser(
        description="Model selector for hugging face models", allow_abbrev=False
    )

    # File I/O
    parser.add_argument(
        "--train_file_path",
        type=str,
        help="Input train file path",
    )

    parser.add_argument(
        "--validation_file_path",
        default=None,
        type=str,
        help="Input validation file path",
    )

    parser.add_argument(
        "--generated_train_file_path",
        type=Path,
        default=None,
        help="file to save the generated training data",
    )

    parser.add_argument(
        "--generated_validation_file_path",
        type=Path,
        default=None,
        help="file to save the generated validation data",
    )

    # add optional data-generator params
    parser.add_argument(
        "--teacher_model_endpoint_name",
        type=str,
        required=False,
        help="Teacher model endpoint name",
    )
    parser.add_argument(
        "--teacher_model_endpoint_url",
        type=str,
        required=False,
        help="Teacher model endpoint URL",
    )
    parser.add_argument(
        "--teacher_model_endpoint_key",
        type=str,
        required=False,
        help="Teacher model endpoint key",
    )
    parser.add_argument(
        "--teacher_model_max_new_tokens",
        type=int,
        required=False,
        default=DEFAULT_MAX_NEW_TOKENS,
        help="Teacher model max_tokens parameter",
    )
    parser.add_argument(
        "--teacher_model_temperature",
        type=float,
        required=False,
        default=DEFAULT_TEMPERATURE,
        help="Teacher model temperature parameter",
    )
    parser.add_argument(
        "--teacher_model_top_p",
        type=float,
        required=False,
        default=DEFAULT_TOP_P,
        help="Teacher model top-p parameter",
    )
    parser.add_argument(
        "--teacher_model_frequency_penalty",
        type=float,
        required=False,
        help="Teacher model frequency parameter",
    )
    parser.add_argument(
        "--teacher_model_presence_penalty",
        type=float,
        required=False,
        help="Teacher model presense penalty",
    )
    parser.add_argument(
        "--teacher_model_stop", type=str, required=False, help="Teacher model stop "
    )
    parser.add_argument(
        "--request_batch_size",
        type=int,
        default=DEFAULT_REQUEST_BATCH_SIZE,
        required=False,
        help="No of data records to process at a time.",
    )
    parser.add_argument(
        "--min_endpoint_success_ratio",
        type=float,
        required=False,
        default=DEFAULT_SUCCESS_RATIO,
        help=(
            f"The minimum value of "
            "(successful_requests / total_requests) required for classifying inference as successful. "
            "If (successful_requests / total_requests) < min_endpoint_success_ratio, "
            "the experiment will be marked as failed. "
            f"By default it is {DEFAULT_SUCCESS_RATIO}. "
            "(0 means all requests are allowed to fail while 1 means no request should fail.)"
        ),
    )

    parser.add_argument(
        "--enable_chain_of_thought",
        type=str,
        required=False,
        default="false",
        help="This enables Chain of Thought",
    )

    parser.add_argument(
        "--enable_chain_of_density",
        type=str,
        required=False,
        default="false",
        help="This enables Chain of Density for Summarization",
    )

    parser.add_argument(
        "--max_len_summary",
        type=int,
        required=False,
        default=DEFAULT_MAX_LEN_SUMMARY,
        help="Maximum word count for text summarization ",
    )

    parser.add_argument(
        "--data_generation_task_type",
        type=str,
        required=True,
        help="""Data generation task type. Supported values are: