def _construct_spark_submit_command()

in src/smspark/cli.py [0:0]

• 18 lines of code
• 10 McCabe index (conditional complexity)

def _construct_spark_submit_command(spark_opts: Dict[str, Any], app_and_app_arguments: Sequence[str]) -> str:
    """Construct a spark-submit command from smspark-submit options, app, and app arguments.

    Args:
        spark_opts (Sequence[str]): A sequence of strings to pass to spark-submit.
        app_and_app_arguments (Sequence[str]): A sequence of strings consisting of the application jar or file,
                                        and application arguments.
    Returns:
        str: The string to be used as the spark-submit command.
    """
    spark_options_list = []
    for key, val in spark_opts.items():
        # "-v" or "--verbose" is an option flag, unlike the long options, which require a value.
        if key == "verbose" and val is True:
            spark_options_list.append("--verbose")
            continue
        # Undo collision between python built-in "class" and spark "--class" option.
        if key == "class_":
            key = "class"

        # Python converts hyphenated options ("py-files") to underscored attributes ("py_files"). Undo underscores.
        if "_" in key:
            key = key.replace("_", "-")
        if val:
            spark_options_list.append(f"--{key}")
            spark_options_list.append(val)
    cmd = ["spark-submit", "--master", "yarn", "--deploy-mode", "client"]
    cmd.extend(spark_options_list)
    cmd.extend(app_and_app_arguments)
    cmd_string = " ".join(shlex.quote(c) for c in cmd)
    return cmd_string