python/pipelines/automl_tabular_pl_v1.yaml [5643:6023]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
          "\", \"--lro_job_info=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/lro\",
          \"--error_file_path=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb\",
          \"--metadata_path=", "{{$.inputs.artifacts[''metadata''].uri}}", "\", \"--materialized_train_split=",
          "{{$.inputs.artifacts[''materialized_train_split''].uri}}", "\", \"--materialized_eval_split=",
          "{{$.inputs.artifacts[''materialized_eval_split''].uri}}", "\", \"--is_distill=",
          "{{$.inputs.parameters[''run_distillation'']}}", "\", \"--tuning_result_output_path=",
          "{{$.outputs.artifacts[''tuning_result_output''].uri}}", "\", \"--kms_key_name=",
          "{{$.inputs.parameters[''encryption_spec_key_name'']}}", "\", \"--gcp_resources_path=",
          "{{$.outputs.parameters[''gcp_resources''].output_file}}", "\", \"--execution_metrics_path=",
          "{{$.outputs.parameters[''execution_metrics''].output_file}}", "\", \"--use_json=true\",
          \"--log_level=ERROR\", \"--executor_input={{$.json_escape[1]}}\"]}}]}}"]}'
        command:
        - python3
        - -u
        - -m
        - google_cloud_pipeline_components.container.v1.custom_job.launcher
        image: gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.32
    exec-automl-tabular-transform:
      container:
        args:
        - --type
        - CustomJob
        - --project
        - '{{$.inputs.parameters[''project'']}}'
        - --location
        - '{{$.inputs.parameters[''location'']}}'
        - --gcp_resources
        - '{{$.outputs.parameters[''gcp_resources''].output_file}}'
        - --payload
        - '{"Concat": ["{\"display_name\": \"automl-tabular-transform-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}\",
          \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}",
          "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\":
          {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"",
          "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:dev", "\",
          \"args\": [\"transform\", \"--transform_output_artifact_path=", "{{$.outputs.artifacts[''transform_output''].uri}}",
          "\", \"--transform_output_path=", "{{$.inputs.parameters[''root_dir'']}}",
          "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/transform\", \"--materialized_splits_output_path=",
          "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/transform_materialized\",
          \"--metadata_path=", "{{$.inputs.artifacts[''metadata''].uri}}", "\", \"--dataset_schema_path=",
          "{{$.inputs.artifacts[''dataset_schema''].uri}}", "\", \"--train_split=",
          "{{$.inputs.artifacts[''train_split''].uri}}", "\", \"--eval_split=", "{{$.inputs.artifacts[''eval_split''].uri}}",
          "\", \"--test_split=", "{{$.inputs.artifacts[''test_split''].uri}}", "\",
          \"--materialized_train_split=", "{{$.outputs.artifacts[''materialized_train_split''].uri}}",
          "\", \"--materialized_eval_split=", "{{$.outputs.artifacts[''materialized_eval_split''].uri}}",
          "\", \"--materialized_test_split=", "{{$.outputs.artifacts[''materialized_test_split''].uri}}",
          "\", \"--training_schema_path=", "{{$.outputs.artifacts[''training_schema_uri''].uri}}",
          "\", \"--job_name=automl-tabular-transform-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}",
          "\", \"--dataflow_project=", "{{$.inputs.parameters[''project'']}}", "\",
          \"--error_file_path=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb\",
          \"--dataflow_staging_dir=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_staging\",
          \"--dataflow_tmp_dir=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_tmp\",
          \"--dataflow_max_num_workers=", "{{$.inputs.parameters[''dataflow_max_num_workers'']}}",
          "\", \"--dataflow_machine_type=", "{{$.inputs.parameters[''dataflow_machine_type'']}}",
          "\", \"--dataflow_worker_container_image=", "us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:dev",
          "\", \"--dataflow_disk_size_gb=", "{{$.inputs.parameters[''dataflow_disk_size_gb'']}}",
          "\", \"--dataflow_subnetwork_fully_qualified=", "{{$.inputs.parameters[''dataflow_subnetwork'']}}",
          "\", \"--dataflow_use_public_ips=", "{{$.inputs.parameters[''dataflow_use_public_ips'']}}",
          "\", \"--dataflow_kms_key=", "{{$.inputs.parameters[''encryption_spec_key_name'']}}",
          "\", \"--dataflow_service_account=", "{{$.inputs.parameters[''dataflow_service_account'']}}",
          "\", \"--lro_job_info=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/lro\",
          \"--gcp_resources_path=", "{{$.outputs.parameters[''gcp_resources''].output_file}}",
          "\"]}}]}}"]}'
        command:
        - python3
        - -u
        - -m
        - google_cloud_pipeline_components.container.v1.custom_job.launcher
        image: gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.32
    exec-automl-tabular-transform-2:
      container:
        args:
        - --type
        - CustomJob
        - --project
        - '{{$.inputs.parameters[''project'']}}'
        - --location
        - '{{$.inputs.parameters[''location'']}}'
        - --gcp_resources
        - '{{$.outputs.parameters[''gcp_resources''].output_file}}'
        - --payload
        - '{"Concat": ["{\"display_name\": \"automl-tabular-transform-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}\",
          \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}",
          "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\":
          {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"",
          "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:dev", "\",
          \"args\": [\"transform\", \"--transform_output_artifact_path=", "{{$.outputs.artifacts[''transform_output''].uri}}",
          "\", \"--transform_output_path=", "{{$.inputs.parameters[''root_dir'']}}",
          "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/transform\", \"--materialized_splits_output_path=",
          "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/transform_materialized\",
          \"--metadata_path=", "{{$.inputs.artifacts[''metadata''].uri}}", "\", \"--dataset_schema_path=",
          "{{$.inputs.artifacts[''dataset_schema''].uri}}", "\", \"--train_split=",
          "{{$.inputs.artifacts[''train_split''].uri}}", "\", \"--eval_split=", "{{$.inputs.artifacts[''eval_split''].uri}}",
          "\", \"--test_split=", "{{$.inputs.artifacts[''test_split''].uri}}", "\",
          \"--materialized_train_split=", "{{$.outputs.artifacts[''materialized_train_split''].uri}}",
          "\", \"--materialized_eval_split=", "{{$.outputs.artifacts[''materialized_eval_split''].uri}}",
          "\", \"--materialized_test_split=", "{{$.outputs.artifacts[''materialized_test_split''].uri}}",
          "\", \"--training_schema_path=", "{{$.outputs.artifacts[''training_schema_uri''].uri}}",
          "\", \"--job_name=automl-tabular-transform-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}",
          "\", \"--dataflow_project=", "{{$.inputs.parameters[''project'']}}", "\",
          \"--error_file_path=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb\",
          \"--dataflow_staging_dir=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_staging\",
          \"--dataflow_tmp_dir=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_tmp\",
          \"--dataflow_max_num_workers=", "{{$.inputs.parameters[''dataflow_max_num_workers'']}}",
          "\", \"--dataflow_machine_type=", "{{$.inputs.parameters[''dataflow_machine_type'']}}",
          "\", \"--dataflow_worker_container_image=", "us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:dev",
          "\", \"--dataflow_disk_size_gb=", "{{$.inputs.parameters[''dataflow_disk_size_gb'']}}",
          "\", \"--dataflow_subnetwork_fully_qualified=", "{{$.inputs.parameters[''dataflow_subnetwork'']}}",
          "\", \"--dataflow_use_public_ips=", "{{$.inputs.parameters[''dataflow_use_public_ips'']}}",
          "\", \"--dataflow_kms_key=", "{{$.inputs.parameters[''encryption_spec_key_name'']}}",
          "\", \"--dataflow_service_account=", "{{$.inputs.parameters[''dataflow_service_account'']}}",
          "\", \"--lro_job_info=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/lro\",
          \"--gcp_resources_path=", "{{$.outputs.parameters[''gcp_resources''].output_file}}",
          "\"]}}]}}"]}'
        command:
        - python3
        - -u
        - -m
        - google_cloud_pipeline_components.container.v1.custom_job.launcher
        image: gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.32
    exec-bool-identity:
      container:
        args:
        - --executor_input
        - '{{$}}'
        - --function_to_execute
        - _bool_identity
        command:
        - sh
        - -ec
        - 'program_path=$(mktemp -d)

          printf "%s" "$0" > "$program_path/ephemeral_component.py"

          python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"

          '
        - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
          \ *\n\ndef _bool_identity(value: bool) -> str:\n  \"\"\"Returns boolean\
          \ value.\n\n  Args:\n    value: Boolean value to return\n\n  Returns:\n\
          \    Boolean value.\n  \"\"\"\n  return 'true' if value else 'false'\n\n"
        image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:dev
    exec-bool-identity-2:
      container:
        args:
        - --executor_input
        - '{{$}}'
        - --function_to_execute
        - _bool_identity
        command:
        - sh
        - -ec
        - 'program_path=$(mktemp -d)

          printf "%s" "$0" > "$program_path/ephemeral_component.py"

          python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"

          '
        - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
          \ *\n\ndef _bool_identity(value: bool) -> str:\n  \"\"\"Returns boolean\
          \ value.\n\n  Args:\n    value: Boolean value to return\n\n  Returns:\n\
          \    Boolean value.\n  \"\"\"\n  return 'true' if value else 'false'\n\n"
        image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:dev
    exec-bool-identity-3:
      container:
        args:
        - --executor_input
        - '{{$}}'
        - --function_to_execute
        - _bool_identity
        command:
        - sh
        - -ec
        - 'program_path=$(mktemp -d)

          printf "%s" "$0" > "$program_path/ephemeral_component.py"

          python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"

          '
        - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
          \ *\n\ndef _bool_identity(value: bool) -> str:\n  \"\"\"Returns boolean\
          \ value.\n\n  Args:\n    value: Boolean value to return\n\n  Returns:\n\
          \    Boolean value.\n  \"\"\"\n  return 'true' if value else 'false'\n\n"
        image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:dev
    exec-calculate-training-parameters:
      container:
        args:
        - --executor_input
        - '{{$}}'
        - --function_to_execute
        - _calculate_training_parameters
        command:
        - sh
        - -ec
        - 'program_path=$(mktemp -d)

          printf "%s" "$0" > "$program_path/ephemeral_component.py"

          python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"

          '
        - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
          \ *\n\ndef _calculate_training_parameters(\n    stage_1_num_parallel_trials:\
          \ int,\n    train_budget_milli_node_hours: float,\n    stage_2_num_parallel_trials:\
          \ int,\n    run_distillation: bool,\n    is_skip_architecture_search: bool\
          \ = False,\n    fast_testing: bool = False,\n) -> NamedTuple('Outputs',\
          \ [\n    ('stage_1_deadline_hours', float), ('stage_1_num_selected_trials',\
          \ int),\n    ('stage_1_single_run_max_secs', int), ('stage_2_deadline_hours',\
          \ float),\n    ('stage_2_single_run_max_secs', int),\n    ('distill_stage_1_deadline_hours',\
          \ float), ('reduce_search_space_mode', str)\n]):\n  \"\"\"Calculates training\
          \ parameters.\n\n  Args:\n    stage_1_num_parallel_trials: Number of parallel\
          \ trails for stage 1.\n    train_budget_milli_node_hours: The train budget\
          \ of creating this model,\n      expressed in milli node hours i.e. 1,000\
          \ value in this field means 1 node\n      hour.\n    stage_2_num_parallel_trials:\
          \ Number of parallel trails for stage 2.\n    run_distillation: Whether\
          \ to run distill in the training pipeline.\n    is_skip_architecture_search:\
          \ If component is being called in the\n      skip_architecture_search pipeline.\n\
          \    fast_testing: Internal flag used for presubmit tests.\n\n  Returns:\n\
          \      stage_1_deadline_hours: Maximum number of hours to run stage 1.\n\
          \      stage_1_num_selected_trials: Number of selected trails for stage\
          \ 1.\n      stage_1_single_run_max_secs: Maximum number seconds to for a\
          \ single stage\n      1\n        training trial.\n      stage_2_deadline_hours:\
          \ Maximum number of hours to run stage 2.\n      stage_2_single_run_max_secs:\
          \ Maximum number seconds to for a single stage\n      2\n      training\
          \ trial.\n      distill_stage_1_deadline_hours: Maximum number of hours\
          \ to run stage 1 for\n      the model distillation.\n      reduce_search_space_mode:\
          \ The reduce search space mode. Possible values:\n      minimal, regular,\
          \ full.\n  \"\"\"\n  # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\
          \  import collections\n  import math\n  # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\
          \  num_folds = 5\n  distill_total_trials = 100\n\n  stage_1_deadline_hours\
          \ = -1.0\n  stage_1_num_selected_trials = -1\n  stage_1_single_run_max_secs\
          \ = -1\n  stage_2_deadline_hours = -1.0\n  stage_2_single_run_max_secs =\
          \ -1\n  distill_stage_1_deadline_hours = 1.0\n  reduce_search_space_mode\
          \ = 'regular'\n\n  if is_skip_architecture_search:\n    stage_2_deadline_hours\
          \ = train_budget_milli_node_hours / 1000.0\n    stage_2_single_run_max_secs\
          \ = int(stage_2_deadline_hours * 3600.0 / 1.3)\n  else:\n    hours = float(train_budget_milli_node_hours)\
          \ / 1000.0\n    multiplier = stage_1_num_parallel_trials * hours / 500.0\n\
          \    stage_1_single_run_max_secs = int(math.sqrt(multiplier) * 2400.0)\n\
          \    phase_2_rounds = int(\n        math.sqrt(multiplier) * 100 / stage_2_num_parallel_trials\
          \ + 0.5)\n    if phase_2_rounds < 1:\n      phase_2_rounds = 1\n\n    #\
          \ All of magic number \"1.3\" above is because the trial doesn't\n    #\
          \ always finish in time_per_trial. 1.3 is an empirical safety margin here.\n\
          \    stage_1_deadline_secs = int(hours * 3600.0 - 1.3 *\n              \
          \                  stage_1_single_run_max_secs * phase_2_rounds)\n\n   \
          \ if stage_1_deadline_secs < hours * 3600.0 * 0.5:\n      stage_1_deadline_secs\
          \ = int(hours * 3600.0 * 0.5)\n      # Phase 1 deadline is the same as phase\
          \ 2 deadline in this case. Phase 2\n      # can't finish in time after the\
          \ deadline is cut, so adjust the time per\n      # trial to meet the deadline.\n\
          \      stage_1_single_run_max_secs = int(stage_1_deadline_secs /\n     \
          \                                   (1.3 * phase_2_rounds))\n\n    reduce_search_space_mode\
          \ = 'minimal'\n    if multiplier > 2:\n      reduce_search_space_mode =\
          \ 'regular'\n    if multiplier > 4:\n      reduce_search_space_mode = 'full'\n\
          \n    # Stage 2 number of trials is stage_1_num_selected_trials *\n    #\
          \ num_folds, which should be equal to phase_2_rounds *\n    # stage_2_num_parallel_trials.\
          \ Use this information to calculate\n    # stage_1_num_selected_trials:\n\
          \    stage_1_num_selected_trials = int(phase_2_rounds *\n              \
          \                        stage_2_num_parallel_trials / num_folds)\n    stage_1_deadline_hours\
          \ = stage_1_deadline_secs / 3600.0\n\n    stage_2_deadline_hours = hours\
          \ - stage_1_deadline_hours\n    stage_2_single_run_max_secs = stage_1_single_run_max_secs\n\
          \n    if run_distillation:\n      # All of magic number \"1.3\" above is\
          \ because the trial doesn't always\n      # finish in time_per_trial. 1.3\
          \ is an empirical safety margin here.\n      distill_stage_1_deadline_hours\
          \ = math.ceil(\n          float(distill_total_trials) / stage_1_num_parallel_trials\n\
          \      ) * stage_1_single_run_max_secs * 1.3 / 3600.0\n\n  if fast_testing:\n\
          \    distill_stage_1_deadline_hours = 0.2\n    stage_1_deadline_hours =\
          \ 0.2\n    stage_1_single_run_max_secs = 1\n    stage_2_deadline_hours =\
          \ 0.2\n    stage_2_single_run_max_secs = 1\n\n  return collections.namedtuple('Outputs',\
          \ [\n      'stage_1_deadline_hours', 'stage_1_num_selected_trials',\n  \
          \    'stage_1_single_run_max_secs', 'stage_2_deadline_hours',\n      'stage_2_single_run_max_secs',\
          \ 'distill_stage_1_deadline_hours',\n      'reduce_search_space_mode'\n\
          \  ])(stage_1_deadline_hours, stage_1_num_selected_trials,\n     stage_1_single_run_max_secs,\
          \ stage_2_deadline_hours,\n     stage_2_single_run_max_secs, distill_stage_1_deadline_hours,\n\
          \     reduce_search_space_mode)\n\n"
        image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:dev
    exec-calculate-training-parameters-2:
      container:
        args:
        - --executor_input
        - '{{$}}'
        - --function_to_execute
        - _calculate_training_parameters
        command:
        - sh
        - -ec
        - 'program_path=$(mktemp -d)

          printf "%s" "$0" > "$program_path/ephemeral_component.py"

          python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"

          '
        - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
          \ *\n\ndef _calculate_training_parameters(\n    stage_1_num_parallel_trials:\
          \ int,\n    train_budget_milli_node_hours: float,\n    stage_2_num_parallel_trials:\
          \ int,\n    run_distillation: bool,\n    is_skip_architecture_search: bool\
          \ = False,\n    fast_testing: bool = False,\n) -> NamedTuple('Outputs',\
          \ [\n    ('stage_1_deadline_hours', float), ('stage_1_num_selected_trials',\
          \ int),\n    ('stage_1_single_run_max_secs', int), ('stage_2_deadline_hours',\
          \ float),\n    ('stage_2_single_run_max_secs', int),\n    ('distill_stage_1_deadline_hours',\
          \ float), ('reduce_search_space_mode', str)\n]):\n  \"\"\"Calculates training\
          \ parameters.\n\n  Args:\n    stage_1_num_parallel_trials: Number of parallel\
          \ trails for stage 1.\n    train_budget_milli_node_hours: The train budget\
          \ of creating this model,\n      expressed in milli node hours i.e. 1,000\
          \ value in this field means 1 node\n      hour.\n    stage_2_num_parallel_trials:\
          \ Number of parallel trails for stage 2.\n    run_distillation: Whether\
          \ to run distill in the training pipeline.\n    is_skip_architecture_search:\
          \ If component is being called in the\n      skip_architecture_search pipeline.\n\
          \    fast_testing: Internal flag used for presubmit tests.\n\n  Returns:\n\
          \      stage_1_deadline_hours: Maximum number of hours to run stage 1.\n\
          \      stage_1_num_selected_trials: Number of selected trails for stage\
          \ 1.\n      stage_1_single_run_max_secs: Maximum number seconds to for a\
          \ single stage\n      1\n        training trial.\n      stage_2_deadline_hours:\
          \ Maximum number of hours to run stage 2.\n      stage_2_single_run_max_secs:\
          \ Maximum number seconds to for a single stage\n      2\n      training\
          \ trial.\n      distill_stage_1_deadline_hours: Maximum number of hours\
          \ to run stage 1 for\n      the model distillation.\n      reduce_search_space_mode:\
          \ The reduce search space mode. Possible values:\n      minimal, regular,\
          \ full.\n  \"\"\"\n  # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\
          \  import collections\n  import math\n  # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\
          \  num_folds = 5\n  distill_total_trials = 100\n\n  stage_1_deadline_hours\
          \ = -1.0\n  stage_1_num_selected_trials = -1\n  stage_1_single_run_max_secs\
          \ = -1\n  stage_2_deadline_hours = -1.0\n  stage_2_single_run_max_secs =\
          \ -1\n  distill_stage_1_deadline_hours = 1.0\n  reduce_search_space_mode\
          \ = 'regular'\n\n  if is_skip_architecture_search:\n    stage_2_deadline_hours\
          \ = train_budget_milli_node_hours / 1000.0\n    stage_2_single_run_max_secs\
          \ = int(stage_2_deadline_hours * 3600.0 / 1.3)\n  else:\n    hours = float(train_budget_milli_node_hours)\
          \ / 1000.0\n    multiplier = stage_1_num_parallel_trials * hours / 500.0\n\
          \    stage_1_single_run_max_secs = int(math.sqrt(multiplier) * 2400.0)\n\
          \    phase_2_rounds = int(\n        math.sqrt(multiplier) * 100 / stage_2_num_parallel_trials\
          \ + 0.5)\n    if phase_2_rounds < 1:\n      phase_2_rounds = 1\n\n    #\
          \ All of magic number \"1.3\" above is because the trial doesn't\n    #\
          \ always finish in time_per_trial. 1.3 is an empirical safety margin here.\n\
          \    stage_1_deadline_secs = int(hours * 3600.0 - 1.3 *\n              \
          \                  stage_1_single_run_max_secs * phase_2_rounds)\n\n   \
          \ if stage_1_deadline_secs < hours * 3600.0 * 0.5:\n      stage_1_deadline_secs\
          \ = int(hours * 3600.0 * 0.5)\n      # Phase 1 deadline is the same as phase\
          \ 2 deadline in this case. Phase 2\n      # can't finish in time after the\
          \ deadline is cut, so adjust the time per\n      # trial to meet the deadline.\n\
          \      stage_1_single_run_max_secs = int(stage_1_deadline_secs /\n     \
          \                                   (1.3 * phase_2_rounds))\n\n    reduce_search_space_mode\
          \ = 'minimal'\n    if multiplier > 2:\n      reduce_search_space_mode =\
          \ 'regular'\n    if multiplier > 4:\n      reduce_search_space_mode = 'full'\n\
          \n    # Stage 2 number of trials is stage_1_num_selected_trials *\n    #\
          \ num_folds, which should be equal to phase_2_rounds *\n    # stage_2_num_parallel_trials.\
          \ Use this information to calculate\n    # stage_1_num_selected_trials:\n\
          \    stage_1_num_selected_trials = int(phase_2_rounds *\n              \
          \                        stage_2_num_parallel_trials / num_folds)\n    stage_1_deadline_hours\
          \ = stage_1_deadline_secs / 3600.0\n\n    stage_2_deadline_hours = hours\
          \ - stage_1_deadline_hours\n    stage_2_single_run_max_secs = stage_1_single_run_max_secs\n\
          \n    if run_distillation:\n      # All of magic number \"1.3\" above is\
          \ because the trial doesn't always\n      # finish in time_per_trial. 1.3\
          \ is an empirical safety margin here.\n      distill_stage_1_deadline_hours\
          \ = math.ceil(\n          float(distill_total_trials) / stage_1_num_parallel_trials\n\
          \      ) * stage_1_single_run_max_secs * 1.3 / 3600.0\n\n  if fast_testing:\n\
          \    distill_stage_1_deadline_hours = 0.2\n    stage_1_deadline_hours =\
          \ 0.2\n    stage_1_single_run_max_secs = 1\n    stage_2_deadline_hours =\
          \ 0.2\n    stage_2_single_run_max_secs = 1\n\n  return collections.namedtuple('Outputs',\
          \ [\n      'stage_1_deadline_hours', 'stage_1_num_selected_trials',\n  \
          \    'stage_1_single_run_max_secs', 'stage_2_deadline_hours',\n      'stage_2_single_run_max_secs',\
          \ 'distill_stage_1_deadline_hours',\n      'reduce_search_space_mode'\n\
          \  ])(stage_1_deadline_hours, stage_1_num_selected_trials,\n     stage_1_single_run_max_secs,\
          \ stage_2_deadline_hours,\n     stage_2_single_run_max_secs, distill_stage_1_deadline_hours,\n\
          \     reduce_search_space_mode)\n\n"
        image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:dev
    exec-feature-attribution:
      container:
        args:
        - --task
        - explanation
        - --setup_file
        - /setup.py
        - --project_id
        - '{{$.inputs.parameters[''project'']}}'
        - --location
        - '{{$.inputs.parameters[''location'']}}'
        - --batch_prediction_gcs_source
        - '{{$.inputs.artifacts[''predictions_gcs_source''].uri}}'
        - --batch_prediction_format
        - '{{$.inputs.parameters[''predictions_format'']}}'
        - --root_dir
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



python/pipelines/automl_tabular_pl_v2.yaml [6289:6669]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
          "\", \"--lro_job_info=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/lro\",
          \"--error_file_path=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb\",
          \"--metadata_path=", "{{$.inputs.artifacts[''metadata''].uri}}", "\", \"--materialized_train_split=",
          "{{$.inputs.artifacts[''materialized_train_split''].uri}}", "\", \"--materialized_eval_split=",
          "{{$.inputs.artifacts[''materialized_eval_split''].uri}}", "\", \"--is_distill=",
          "{{$.inputs.parameters[''run_distillation'']}}", "\", \"--tuning_result_output_path=",
          "{{$.outputs.artifacts[''tuning_result_output''].uri}}", "\", \"--kms_key_name=",
          "{{$.inputs.parameters[''encryption_spec_key_name'']}}", "\", \"--gcp_resources_path=",
          "{{$.outputs.parameters[''gcp_resources''].output_file}}", "\", \"--execution_metrics_path=",
          "{{$.outputs.parameters[''execution_metrics''].output_file}}", "\", \"--use_json=true\",
          \"--log_level=ERROR\", \"--executor_input={{$.json_escape[1]}}\"]}}]}}"]}'
        command:
        - python3
        - -u
        - -m
        - google_cloud_pipeline_components.container.v1.custom_job.launcher
        image: gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.32
    exec-automl-tabular-transform:
      container:
        args:
        - --type
        - CustomJob
        - --project
        - '{{$.inputs.parameters[''project'']}}'
        - --location
        - '{{$.inputs.parameters[''location'']}}'
        - --gcp_resources
        - '{{$.outputs.parameters[''gcp_resources''].output_file}}'
        - --payload
        - '{"Concat": ["{\"display_name\": \"automl-tabular-transform-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}\",
          \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}",
          "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\":
          {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"",
          "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:dev", "\",
          \"args\": [\"transform\", \"--transform_output_artifact_path=", "{{$.outputs.artifacts[''transform_output''].uri}}",
          "\", \"--transform_output_path=", "{{$.inputs.parameters[''root_dir'']}}",
          "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/transform\", \"--materialized_splits_output_path=",
          "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/transform_materialized\",
          \"--metadata_path=", "{{$.inputs.artifacts[''metadata''].uri}}", "\", \"--dataset_schema_path=",
          "{{$.inputs.artifacts[''dataset_schema''].uri}}", "\", \"--train_split=",
          "{{$.inputs.artifacts[''train_split''].uri}}", "\", \"--eval_split=", "{{$.inputs.artifacts[''eval_split''].uri}}",
          "\", \"--test_split=", "{{$.inputs.artifacts[''test_split''].uri}}", "\",
          \"--materialized_train_split=", "{{$.outputs.artifacts[''materialized_train_split''].uri}}",
          "\", \"--materialized_eval_split=", "{{$.outputs.artifacts[''materialized_eval_split''].uri}}",
          "\", \"--materialized_test_split=", "{{$.outputs.artifacts[''materialized_test_split''].uri}}",
          "\", \"--training_schema_path=", "{{$.outputs.artifacts[''training_schema_uri''].uri}}",
          "\", \"--job_name=automl-tabular-transform-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}",
          "\", \"--dataflow_project=", "{{$.inputs.parameters[''project'']}}", "\",
          \"--error_file_path=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb\",
          \"--dataflow_staging_dir=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_staging\",
          \"--dataflow_tmp_dir=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_tmp\",
          \"--dataflow_max_num_workers=", "{{$.inputs.parameters[''dataflow_max_num_workers'']}}",
          "\", \"--dataflow_machine_type=", "{{$.inputs.parameters[''dataflow_machine_type'']}}",
          "\", \"--dataflow_worker_container_image=", "us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:dev",
          "\", \"--dataflow_disk_size_gb=", "{{$.inputs.parameters[''dataflow_disk_size_gb'']}}",
          "\", \"--dataflow_subnetwork_fully_qualified=", "{{$.inputs.parameters[''dataflow_subnetwork'']}}",
          "\", \"--dataflow_use_public_ips=", "{{$.inputs.parameters[''dataflow_use_public_ips'']}}",
          "\", \"--dataflow_kms_key=", "{{$.inputs.parameters[''encryption_spec_key_name'']}}",
          "\", \"--dataflow_service_account=", "{{$.inputs.parameters[''dataflow_service_account'']}}",
          "\", \"--lro_job_info=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/lro\",
          \"--gcp_resources_path=", "{{$.outputs.parameters[''gcp_resources''].output_file}}",
          "\"]}}]}}"]}'
        command:
        - python3
        - -u
        - -m
        - google_cloud_pipeline_components.container.v1.custom_job.launcher
        image: gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.32
    exec-automl-tabular-transform-2:
      container:
        args:
        - --type
        - CustomJob
        - --project
        - '{{$.inputs.parameters[''project'']}}'
        - --location
        - '{{$.inputs.parameters[''location'']}}'
        - --gcp_resources
        - '{{$.outputs.parameters[''gcp_resources''].output_file}}'
        - --payload
        - '{"Concat": ["{\"display_name\": \"automl-tabular-transform-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}\",
          \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}",
          "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\":
          {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"",
          "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:dev", "\",
          \"args\": [\"transform\", \"--transform_output_artifact_path=", "{{$.outputs.artifacts[''transform_output''].uri}}",
          "\", \"--transform_output_path=", "{{$.inputs.parameters[''root_dir'']}}",
          "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/transform\", \"--materialized_splits_output_path=",
          "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/transform_materialized\",
          \"--metadata_path=", "{{$.inputs.artifacts[''metadata''].uri}}", "\", \"--dataset_schema_path=",
          "{{$.inputs.artifacts[''dataset_schema''].uri}}", "\", \"--train_split=",
          "{{$.inputs.artifacts[''train_split''].uri}}", "\", \"--eval_split=", "{{$.inputs.artifacts[''eval_split''].uri}}",
          "\", \"--test_split=", "{{$.inputs.artifacts[''test_split''].uri}}", "\",
          \"--materialized_train_split=", "{{$.outputs.artifacts[''materialized_train_split''].uri}}",
          "\", \"--materialized_eval_split=", "{{$.outputs.artifacts[''materialized_eval_split''].uri}}",
          "\", \"--materialized_test_split=", "{{$.outputs.artifacts[''materialized_test_split''].uri}}",
          "\", \"--training_schema_path=", "{{$.outputs.artifacts[''training_schema_uri''].uri}}",
          "\", \"--job_name=automl-tabular-transform-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}",
          "\", \"--dataflow_project=", "{{$.inputs.parameters[''project'']}}", "\",
          \"--error_file_path=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb\",
          \"--dataflow_staging_dir=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_staging\",
          \"--dataflow_tmp_dir=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_tmp\",
          \"--dataflow_max_num_workers=", "{{$.inputs.parameters[''dataflow_max_num_workers'']}}",
          "\", \"--dataflow_machine_type=", "{{$.inputs.parameters[''dataflow_machine_type'']}}",
          "\", \"--dataflow_worker_container_image=", "us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:dev",
          "\", \"--dataflow_disk_size_gb=", "{{$.inputs.parameters[''dataflow_disk_size_gb'']}}",
          "\", \"--dataflow_subnetwork_fully_qualified=", "{{$.inputs.parameters[''dataflow_subnetwork'']}}",
          "\", \"--dataflow_use_public_ips=", "{{$.inputs.parameters[''dataflow_use_public_ips'']}}",
          "\", \"--dataflow_kms_key=", "{{$.inputs.parameters[''encryption_spec_key_name'']}}",
          "\", \"--dataflow_service_account=", "{{$.inputs.parameters[''dataflow_service_account'']}}",
          "\", \"--lro_job_info=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/lro\",
          \"--gcp_resources_path=", "{{$.outputs.parameters[''gcp_resources''].output_file}}",
          "\"]}}]}}"]}'
        command:
        - python3
        - -u
        - -m
        - google_cloud_pipeline_components.container.v1.custom_job.launcher
        image: gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.32
    exec-bool-identity:
      container:
        args:
        - --executor_input
        - '{{$}}'
        - --function_to_execute
        - _bool_identity
        command:
        - sh
        - -ec
        - 'program_path=$(mktemp -d)

          printf "%s" "$0" > "$program_path/ephemeral_component.py"

          python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"

          '
        - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
          \ *\n\ndef _bool_identity(value: bool) -> str:\n  \"\"\"Returns boolean\
          \ value.\n\n  Args:\n    value: Boolean value to return\n\n  Returns:\n\
          \    Boolean value.\n  \"\"\"\n  return 'true' if value else 'false'\n\n"
        image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:dev
    exec-bool-identity-2:
      container:
        args:
        - --executor_input
        - '{{$}}'
        - --function_to_execute
        - _bool_identity
        command:
        - sh
        - -ec
        - 'program_path=$(mktemp -d)

          printf "%s" "$0" > "$program_path/ephemeral_component.py"

          python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"

          '
        - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
          \ *\n\ndef _bool_identity(value: bool) -> str:\n  \"\"\"Returns boolean\
          \ value.\n\n  Args:\n    value: Boolean value to return\n\n  Returns:\n\
          \    Boolean value.\n  \"\"\"\n  return 'true' if value else 'false'\n\n"
        image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:dev
    exec-bool-identity-3:
      container:
        args:
        - --executor_input
        - '{{$}}'
        - --function_to_execute
        - _bool_identity
        command:
        - sh
        - -ec
        - 'program_path=$(mktemp -d)

          printf "%s" "$0" > "$program_path/ephemeral_component.py"

          python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"

          '
        - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
          \ *\n\ndef _bool_identity(value: bool) -> str:\n  \"\"\"Returns boolean\
          \ value.\n\n  Args:\n    value: Boolean value to return\n\n  Returns:\n\
          \    Boolean value.\n  \"\"\"\n  return 'true' if value else 'false'\n\n"
        image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:dev
    exec-calculate-training-parameters:
      container:
        args:
        - --executor_input
        - '{{$}}'
        - --function_to_execute
        - _calculate_training_parameters
        command:
        - sh
        - -ec
        - 'program_path=$(mktemp -d)

          printf "%s" "$0" > "$program_path/ephemeral_component.py"

          python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"

          '
        - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
          \ *\n\ndef _calculate_training_parameters(\n    stage_1_num_parallel_trials:\
          \ int,\n    train_budget_milli_node_hours: float,\n    stage_2_num_parallel_trials:\
          \ int,\n    run_distillation: bool,\n    is_skip_architecture_search: bool\
          \ = False,\n    fast_testing: bool = False,\n) -> NamedTuple('Outputs',\
          \ [\n    ('stage_1_deadline_hours', float), ('stage_1_num_selected_trials',\
          \ int),\n    ('stage_1_single_run_max_secs', int), ('stage_2_deadline_hours',\
          \ float),\n    ('stage_2_single_run_max_secs', int),\n    ('distill_stage_1_deadline_hours',\
          \ float), ('reduce_search_space_mode', str)\n]):\n  \"\"\"Calculates training\
          \ parameters.\n\n  Args:\n    stage_1_num_parallel_trials: Number of parallel\
          \ trails for stage 1.\n    train_budget_milli_node_hours: The train budget\
          \ of creating this model,\n      expressed in milli node hours i.e. 1,000\
          \ value in this field means 1 node\n      hour.\n    stage_2_num_parallel_trials:\
          \ Number of parallel trails for stage 2.\n    run_distillation: Whether\
          \ to run distill in the training pipeline.\n    is_skip_architecture_search:\
          \ If component is being called in the\n      skip_architecture_search pipeline.\n\
          \    fast_testing: Internal flag used for presubmit tests.\n\n  Returns:\n\
          \      stage_1_deadline_hours: Maximum number of hours to run stage 1.\n\
          \      stage_1_num_selected_trials: Number of selected trails for stage\
          \ 1.\n      stage_1_single_run_max_secs: Maximum number seconds to for a\
          \ single stage\n      1\n        training trial.\n      stage_2_deadline_hours:\
          \ Maximum number of hours to run stage 2.\n      stage_2_single_run_max_secs:\
          \ Maximum number seconds to for a single stage\n      2\n      training\
          \ trial.\n      distill_stage_1_deadline_hours: Maximum number of hours\
          \ to run stage 1 for\n      the model distillation.\n      reduce_search_space_mode:\
          \ The reduce search space mode. Possible values:\n      minimal, regular,\
          \ full.\n  \"\"\"\n  # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\
          \  import collections\n  import math\n  # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\
          \  num_folds = 5\n  distill_total_trials = 100\n\n  stage_1_deadline_hours\
          \ = -1.0\n  stage_1_num_selected_trials = -1\n  stage_1_single_run_max_secs\
          \ = -1\n  stage_2_deadline_hours = -1.0\n  stage_2_single_run_max_secs =\
          \ -1\n  distill_stage_1_deadline_hours = 1.0\n  reduce_search_space_mode\
          \ = 'regular'\n\n  if is_skip_architecture_search:\n    stage_2_deadline_hours\
          \ = train_budget_milli_node_hours / 1000.0\n    stage_2_single_run_max_secs\
          \ = int(stage_2_deadline_hours * 3600.0 / 1.3)\n  else:\n    hours = float(train_budget_milli_node_hours)\
          \ / 1000.0\n    multiplier = stage_1_num_parallel_trials * hours / 500.0\n\
          \    stage_1_single_run_max_secs = int(math.sqrt(multiplier) * 2400.0)\n\
          \    phase_2_rounds = int(\n        math.sqrt(multiplier) * 100 / stage_2_num_parallel_trials\
          \ + 0.5)\n    if phase_2_rounds < 1:\n      phase_2_rounds = 1\n\n    #\
          \ All of magic number \"1.3\" above is because the trial doesn't\n    #\
          \ always finish in time_per_trial. 1.3 is an empirical safety margin here.\n\
          \    stage_1_deadline_secs = int(hours * 3600.0 - 1.3 *\n              \
          \                  stage_1_single_run_max_secs * phase_2_rounds)\n\n   \
          \ if stage_1_deadline_secs < hours * 3600.0 * 0.5:\n      stage_1_deadline_secs\
          \ = int(hours * 3600.0 * 0.5)\n      # Phase 1 deadline is the same as phase\
          \ 2 deadline in this case. Phase 2\n      # can't finish in time after the\
          \ deadline is cut, so adjust the time per\n      # trial to meet the deadline.\n\
          \      stage_1_single_run_max_secs = int(stage_1_deadline_secs /\n     \
          \                                   (1.3 * phase_2_rounds))\n\n    reduce_search_space_mode\
          \ = 'minimal'\n    if multiplier > 2:\n      reduce_search_space_mode =\
          \ 'regular'\n    if multiplier > 4:\n      reduce_search_space_mode = 'full'\n\
          \n    # Stage 2 number of trials is stage_1_num_selected_trials *\n    #\
          \ num_folds, which should be equal to phase_2_rounds *\n    # stage_2_num_parallel_trials.\
          \ Use this information to calculate\n    # stage_1_num_selected_trials:\n\
          \    stage_1_num_selected_trials = int(phase_2_rounds *\n              \
          \                        stage_2_num_parallel_trials / num_folds)\n    stage_1_deadline_hours\
          \ = stage_1_deadline_secs / 3600.0\n\n    stage_2_deadline_hours = hours\
          \ - stage_1_deadline_hours\n    stage_2_single_run_max_secs = stage_1_single_run_max_secs\n\
          \n    if run_distillation:\n      # All of magic number \"1.3\" above is\
          \ because the trial doesn't always\n      # finish in time_per_trial. 1.3\
          \ is an empirical safety margin here.\n      distill_stage_1_deadline_hours\
          \ = math.ceil(\n          float(distill_total_trials) / stage_1_num_parallel_trials\n\
          \      ) * stage_1_single_run_max_secs * 1.3 / 3600.0\n\n  if fast_testing:\n\
          \    distill_stage_1_deadline_hours = 0.2\n    stage_1_deadline_hours =\
          \ 0.2\n    stage_1_single_run_max_secs = 1\n    stage_2_deadline_hours =\
          \ 0.2\n    stage_2_single_run_max_secs = 1\n\n  return collections.namedtuple('Outputs',\
          \ [\n      'stage_1_deadline_hours', 'stage_1_num_selected_trials',\n  \
          \    'stage_1_single_run_max_secs', 'stage_2_deadline_hours',\n      'stage_2_single_run_max_secs',\
          \ 'distill_stage_1_deadline_hours',\n      'reduce_search_space_mode'\n\
          \  ])(stage_1_deadline_hours, stage_1_num_selected_trials,\n     stage_1_single_run_max_secs,\
          \ stage_2_deadline_hours,\n     stage_2_single_run_max_secs, distill_stage_1_deadline_hours,\n\
          \     reduce_search_space_mode)\n\n"
        image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:dev
    exec-calculate-training-parameters-2:
      container:
        args:
        - --executor_input
        - '{{$}}'
        - --function_to_execute
        - _calculate_training_parameters
        command:
        - sh
        - -ec
        - 'program_path=$(mktemp -d)

          printf "%s" "$0" > "$program_path/ephemeral_component.py"

          python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"

          '
        - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
          \ *\n\ndef _calculate_training_parameters(\n    stage_1_num_parallel_trials:\
          \ int,\n    train_budget_milli_node_hours: float,\n    stage_2_num_parallel_trials:\
          \ int,\n    run_distillation: bool,\n    is_skip_architecture_search: bool\
          \ = False,\n    fast_testing: bool = False,\n) -> NamedTuple('Outputs',\
          \ [\n    ('stage_1_deadline_hours', float), ('stage_1_num_selected_trials',\
          \ int),\n    ('stage_1_single_run_max_secs', int), ('stage_2_deadline_hours',\
          \ float),\n    ('stage_2_single_run_max_secs', int),\n    ('distill_stage_1_deadline_hours',\
          \ float), ('reduce_search_space_mode', str)\n]):\n  \"\"\"Calculates training\
          \ parameters.\n\n  Args:\n    stage_1_num_parallel_trials: Number of parallel\
          \ trails for stage 1.\n    train_budget_milli_node_hours: The train budget\
          \ of creating this model,\n      expressed in milli node hours i.e. 1,000\
          \ value in this field means 1 node\n      hour.\n    stage_2_num_parallel_trials:\
          \ Number of parallel trails for stage 2.\n    run_distillation: Whether\
          \ to run distill in the training pipeline.\n    is_skip_architecture_search:\
          \ If component is being called in the\n      skip_architecture_search pipeline.\n\
          \    fast_testing: Internal flag used for presubmit tests.\n\n  Returns:\n\
          \      stage_1_deadline_hours: Maximum number of hours to run stage 1.\n\
          \      stage_1_num_selected_trials: Number of selected trails for stage\
          \ 1.\n      stage_1_single_run_max_secs: Maximum number seconds to for a\
          \ single stage\n      1\n        training trial.\n      stage_2_deadline_hours:\
          \ Maximum number of hours to run stage 2.\n      stage_2_single_run_max_secs:\
          \ Maximum number seconds to for a single stage\n      2\n      training\
          \ trial.\n      distill_stage_1_deadline_hours: Maximum number of hours\
          \ to run stage 1 for\n      the model distillation.\n      reduce_search_space_mode:\
          \ The reduce search space mode. Possible values:\n      minimal, regular,\
          \ full.\n  \"\"\"\n  # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\
          \  import collections\n  import math\n  # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\
          \  num_folds = 5\n  distill_total_trials = 100\n\n  stage_1_deadline_hours\
          \ = -1.0\n  stage_1_num_selected_trials = -1\n  stage_1_single_run_max_secs\
          \ = -1\n  stage_2_deadline_hours = -1.0\n  stage_2_single_run_max_secs =\
          \ -1\n  distill_stage_1_deadline_hours = 1.0\n  reduce_search_space_mode\
          \ = 'regular'\n\n  if is_skip_architecture_search:\n    stage_2_deadline_hours\
          \ = train_budget_milli_node_hours / 1000.0\n    stage_2_single_run_max_secs\
          \ = int(stage_2_deadline_hours * 3600.0 / 1.3)\n  else:\n    hours = float(train_budget_milli_node_hours)\
          \ / 1000.0\n    multiplier = stage_1_num_parallel_trials * hours / 500.0\n\
          \    stage_1_single_run_max_secs = int(math.sqrt(multiplier) * 2400.0)\n\
          \    phase_2_rounds = int(\n        math.sqrt(multiplier) * 100 / stage_2_num_parallel_trials\
          \ + 0.5)\n    if phase_2_rounds < 1:\n      phase_2_rounds = 1\n\n    #\
          \ All of magic number \"1.3\" above is because the trial doesn't\n    #\
          \ always finish in time_per_trial. 1.3 is an empirical safety margin here.\n\
          \    stage_1_deadline_secs = int(hours * 3600.0 - 1.3 *\n              \
          \                  stage_1_single_run_max_secs * phase_2_rounds)\n\n   \
          \ if stage_1_deadline_secs < hours * 3600.0 * 0.5:\n      stage_1_deadline_secs\
          \ = int(hours * 3600.0 * 0.5)\n      # Phase 1 deadline is the same as phase\
          \ 2 deadline in this case. Phase 2\n      # can't finish in time after the\
          \ deadline is cut, so adjust the time per\n      # trial to meet the deadline.\n\
          \      stage_1_single_run_max_secs = int(stage_1_deadline_secs /\n     \
          \                                   (1.3 * phase_2_rounds))\n\n    reduce_search_space_mode\
          \ = 'minimal'\n    if multiplier > 2:\n      reduce_search_space_mode =\
          \ 'regular'\n    if multiplier > 4:\n      reduce_search_space_mode = 'full'\n\
          \n    # Stage 2 number of trials is stage_1_num_selected_trials *\n    #\
          \ num_folds, which should be equal to phase_2_rounds *\n    # stage_2_num_parallel_trials.\
          \ Use this information to calculate\n    # stage_1_num_selected_trials:\n\
          \    stage_1_num_selected_trials = int(phase_2_rounds *\n              \
          \                        stage_2_num_parallel_trials / num_folds)\n    stage_1_deadline_hours\
          \ = stage_1_deadline_secs / 3600.0\n\n    stage_2_deadline_hours = hours\
          \ - stage_1_deadline_hours\n    stage_2_single_run_max_secs = stage_1_single_run_max_secs\n\
          \n    if run_distillation:\n      # All of magic number \"1.3\" above is\
          \ because the trial doesn't always\n      # finish in time_per_trial. 1.3\
          \ is an empirical safety margin here.\n      distill_stage_1_deadline_hours\
          \ = math.ceil(\n          float(distill_total_trials) / stage_1_num_parallel_trials\n\
          \      ) * stage_1_single_run_max_secs * 1.3 / 3600.0\n\n  if fast_testing:\n\
          \    distill_stage_1_deadline_hours = 0.2\n    stage_1_deadline_hours =\
          \ 0.2\n    stage_1_single_run_max_secs = 1\n    stage_2_deadline_hours =\
          \ 0.2\n    stage_2_single_run_max_secs = 1\n\n  return collections.namedtuple('Outputs',\
          \ [\n      'stage_1_deadline_hours', 'stage_1_num_selected_trials',\n  \
          \    'stage_1_single_run_max_secs', 'stage_2_deadline_hours',\n      'stage_2_single_run_max_secs',\
          \ 'distill_stage_1_deadline_hours',\n      'reduce_search_space_mode'\n\
          \  ])(stage_1_deadline_hours, stage_1_num_selected_trials,\n     stage_1_single_run_max_secs,\
          \ stage_2_deadline_hours,\n     stage_2_single_run_max_secs, distill_stage_1_deadline_hours,\n\
          \     reduce_search_space_mode)\n\n"
        image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:dev
    exec-feature-attribution:
      container:
        args:
        - --task
        - explanation
        - --setup_file
        - /setup.py
        - --project_id
        - '{{$.inputs.parameters[''project'']}}'
        - --location
        - '{{$.inputs.parameters[''location'']}}'
        - --batch_prediction_gcs_source
        - '{{$.inputs.artifacts[''predictions_gcs_source''].uri}}'
        - --batch_prediction_format
        - '{{$.inputs.parameters[''predictions_format'']}}'
        - --root_dir
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



