python-batch/cuda_pubsub/nvidia-python.yaml

#  Copyright 2023 Google LLC
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
project_id: "project_id"
region: "us-central1"

volumes:
  - {bucket_name: "<my_bucket>", gcs_path:  "/mnt/disks/work"}


job_prefix: 'nvidia-python-'
task_count: 12 #optional
parallelism: 4 #optional
task_count_per_node: 4 #optional

machine_type: "n1-standard-4"
accelerator:
 type: "nvidia-tesla-t4"
 count: 2
install_gpu_drivers: y
cpu_milli: 1000 #optional
memory_mib: 102400 #optional

container: 
  image_uri: "us-central1-docker.pkg.dev/<my_project>/cuda/pybatch:latest" 
  entry_point: "/bin/bash"
  commands: 
    - -c
    - "
       export PID=$(gcloud config get-value project);
       echo python3 /tmp/pull_fifo_id.py --project_id=$PID --job_id=$TOPIC_ID;
       export FIFO_ID=$(python3 /tmp/pull_fifo_id.py --project_id=$PID --job_id=$TOPIC_ID);
       echo Obtained FIFO_ID: --- $FIFO_ID ---;
       if [ -z \"$FIFO_ID\" ]; then   echo FIFO NOT DEFINED -- exiting;   exit 0; fi;
       echo Running FIFO_ID:  $FIFO_ID;
       mkdir -p /mnt/disks/work/dir_${BATCH_JOB_ID}; 
       /tmp/simpleCUFFT_2d_MGPU >> /mnt/disks/work/dir_${BATCH_JOB_ID}/out_${FIFO_ID}.txt
      "

labels: 
  env: "container"
  type: "nvidia"