llama2-finetuning-slurm/hpc-slurm-llama2.yaml [27:143]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
  instance_image_custom: true
  disk_size_gb: 200


# Documentation for each of the modules used below can be found at
# https://github.com/GoogleCloudPlatform/hpc-toolkit/blob/main/modules/README.md

deployment_groups:
- group: enable_apis
  modules:

  - id: enable_apis
    source: community/modules/project/service-enablement
    settings:
      gcp_service_list: [
        "cloudresourcemanager.googleapis.com",
        "stackdriver.googleapis.com",
        "iam.googleapis.com",
        "logging.googleapis.com",
        "compute.googleapis.com"
      ]
- group: setup
  modules:

  ## Monitoring
  - id: hpc_dash
    source: modules/monitoring/dashboard
    settings:
      title: HPC
  - id: gpu_dash
    source: modules/monitoring/dashboard
    settings:
      title: GPU
      base_dashboard: Empty
      widgets:
      - |
          {
            "title": "GPU Memory Utilization",
            "xyChart": {
              "dataSets": [
                {
                  "timeSeriesQuery": {
                    "timeSeriesFilter": {
                      "filter": "metric.type=\"agent.googleapis.com/gpu/memory/bytes_used\" resource.type=\"gce_instance\"",
                      "aggregation": {
                        "perSeriesAligner": "ALIGN_MEAN",
                        "crossSeriesReducer": "REDUCE_NONE",
                        "groupByFields": []
                      }
                    }
                  },
                  "plotType": "LINE",
                  "targetAxis": "Y1",
                  "minAlignmentPeriod": "60s"
                }
              ],
              "chartOptions": {
                "mode": "COLOR",
                "displayHorizontal": false
              },
              "thresholds": [],
              "yAxis": {
                "scale": "LINEAR"
              }
            }
          }
      - |
          {
            "title": "GPU Utilization",
            "xyChart": {
              "dataSets": [
                {
                  "timeSeriesQuery": {
                    "prometheusQuery": "avg_over_time(agent_googleapis_com:gpu_utilization{monitored_resource=\"gce_instance\"}[${__interval}])"
                  },
                  "plotType": "LINE",
                  "targetAxis": "Y1"
                }
              ],
              "chartOptions": {
                "mode": "COLOR",
                "displayHorizontal": false
              },
              "thresholds": [],
              "yAxis": {
                "scale": "LINEAR"
              }
            }
          }

  ## network
  - id: network1
    source: modules/network/vpc

  ## Filesystems
  - id: homefs
    source: community/modules/file-system/nfs-server
    use: [network1]
    settings:
      local_mounts: [/home]
      disk_size: 2560
      instance_image:  
        project: "cloud-hpc-image-public"
        family:  "hpc-rocky-linux-8"

  - id: data_bucket
    source: community/modules/file-system/cloud-storage-bucket
    settings:
      name_prefix: $(vars.bucket_model)
      random_suffix: true
      force_destroy: true
      local_mount: /data_bucket
      mount_options: defaults,_netdev,implicit_dirs,allow_other,dir_mode=0777,file_mode=766

  - id: move_files
    source: ./files
    use: [data_bucket]
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



qwiklabs/llama2-finetuning-slurm/hpc-slurm-llama2.yaml [28:144]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
  instance_image_custom: true
  disk_size_gb: 200


# Documentation for each of the modules used below can be found at
# https://github.com/GoogleCloudPlatform/hpc-toolkit/blob/main/modules/README.md

deployment_groups:
- group: enable_apis
  modules:

  - id: enable_apis
    source: community/modules/project/service-enablement
    settings:
      gcp_service_list: [
        "cloudresourcemanager.googleapis.com",
        "stackdriver.googleapis.com",
        "iam.googleapis.com",
        "logging.googleapis.com",
        "compute.googleapis.com"
      ]
- group: setup
  modules:

  ## Monitoring
  - id: hpc_dash
    source: modules/monitoring/dashboard
    settings:
      title: HPC
  - id: gpu_dash
    source: modules/monitoring/dashboard
    settings:
      title: GPU
      base_dashboard: Empty
      widgets:
      - |
          {
            "title": "GPU Memory Utilization",
            "xyChart": {
              "dataSets": [
                {
                  "timeSeriesQuery": {
                    "timeSeriesFilter": {
                      "filter": "metric.type=\"agent.googleapis.com/gpu/memory/bytes_used\" resource.type=\"gce_instance\"",
                      "aggregation": {
                        "perSeriesAligner": "ALIGN_MEAN",
                        "crossSeriesReducer": "REDUCE_NONE",
                        "groupByFields": []
                      }
                    }
                  },
                  "plotType": "LINE",
                  "targetAxis": "Y1",
                  "minAlignmentPeriod": "60s"
                }
              ],
              "chartOptions": {
                "mode": "COLOR",
                "displayHorizontal": false
              },
              "thresholds": [],
              "yAxis": {
                "scale": "LINEAR"
              }
            }
          }
      - |
          {
            "title": "GPU Utilization",
            "xyChart": {
              "dataSets": [
                {
                  "timeSeriesQuery": {
                    "prometheusQuery": "avg_over_time(agent_googleapis_com:gpu_utilization{monitored_resource=\"gce_instance\"}[${__interval}])"
                  },
                  "plotType": "LINE",
                  "targetAxis": "Y1"
                }
              ],
              "chartOptions": {
                "mode": "COLOR",
                "displayHorizontal": false
              },
              "thresholds": [],
              "yAxis": {
                "scale": "LINEAR"
              }
            }
          }

  ## network
  - id: network1
    source: modules/network/vpc

  ## Filesystems
  - id: homefs
    source: community/modules/file-system/nfs-server
    use: [network1]
    settings:
      local_mounts: [/home]
      disk_size: 2560
      instance_image:  
        project: "cloud-hpc-image-public"
        family:  "hpc-rocky-linux-8"

  - id: data_bucket
    source: community/modules/file-system/cloud-storage-bucket
    settings:
      name_prefix: $(vars.bucket_model)
      random_suffix: true
      force_destroy: true
      local_mount: /data_bucket
      mount_options: defaults,_netdev,implicit_dirs,allow_other,dir_mode=0777,file_mode=766

  - id: move_files
    source: ./files
    use: [data_bucket]
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



