read_stall_retry/go-sdk-job.yaml

# Copyright 2025 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

apiVersion: batch/v1
kind: Job
metadata:
  name: slowenvironment-100-nodes-parallel
spec:
  parallelism: 100
  completions: 100
  completionMode: Indexed
  template:
    metadata:
      labels:
        app: benchmark-job
    spec:
      restartPolicy: Never
      serviceAccountName: vipinydv-go-sdk
      hostNetwork: true
      affinity:
        podAntiAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
          - labelSelector:
              matchExpressions:
              - key: app
                operator: In
                values:
                - benchmark-job
            topologyKey: "kubernetes.io/hostname"
      containers:
      - name: go-sdk-tail-read-latency
        image: ubuntu:20.04
        env:
        - name: POD_NAME
          valueFrom:
            fieldRef:
              fieldPath: metadata.name
        - name: JOB_NAME
          valueFrom:
            fieldRef:
              fieldPath: metadata.labels['job-name']
        command:
        - /bin/bash
        - -c
        - |
          set -x
          set -e

          export DEBIAN_FRONTEND=noninteractive

          # Install necessary dependencies
          apt-get update
          apt-get install -y curl sudo gnupg git wget

          # install go
          architecture=$(dpkg --print-architecture)
          wget -O go_tar.tar.gz https://go.dev/dl/go1.23.5.linux-${architecture}.tar.gz
          sudo tar -C /usr/local -xzf go_tar.tar.gz
          export PATH=${PATH}:/usr/local/go/bin

          # Clone the custom go client repository and run the benchmark
          git clone -b read-stall-retry-scale-test https://github.com/raj-prince/custom-go-client-benchmark.git
          cd custom-go-client-benchmark
          go run ./ --worker=32 --bucket=vipin-us-west4 --project=gcs-fuse-test --bucket-dir=1B --with-read-stall-timeout=false --output-bucket-path=vipin-logs-and-metrics/metrics/${JOB_NAME}/${POD_NAME}-metrics.csv --total-files-to-read=18000000 >> output-log.txt 2>&1
          echo "********* Run Complete ***********"

          # Set up Google Cloud SDK for future use (if needed)
          sudo echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] http://packages.cloud.google.com/apt cloud-sdk main" | sudo tee -a /etc/apt/sources.list.d/google-cloud-sdk.list
          sudo curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key --keyring /usr/share/keyrings/cloud.google.gpg add -
          apt-get update
          apt-get install -y google-cloud-sdk

          gcloud storage cp output-log.txt gs://vipin-logs-and-metrics/logs/${JOB_NAME}/${POD_NAME}-logs.txt
          echo "****** Upload Csv complete *******"
---
apiVersion: batch/v1
kind: Job
metadata:
  name: slowenvironment-100-nodes-99p-readstall-parallel
spec:
  parallelism: 100
  completions: 100
  completionMode: Indexed
  template:
    metadata:
      labels:
        app: benchmark-job
    spec:
      restartPolicy: Never
      serviceAccountName: vipinydv-go-sdk
      hostNetwork: true
      affinity:
        podAntiAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
          - labelSelector:
              matchExpressions:
              - key: app
                operator: In
                values:
                - benchmark-job
            topologyKey: "kubernetes.io/hostname"
      containers:
      - name: go-sdk-tail-read-latency
        image: ubuntu:20.04
        env:
        - name: POD_NAME
          valueFrom:
            fieldRef:
              fieldPath: metadata.name
        - name: JOB_NAME
          valueFrom:
            fieldRef:
              fieldPath: metadata.labels['job-name']
        command:
        - /bin/bash
        - -c
        - |
          set -x
          set -e

          export DEBIAN_FRONTEND=noninteractive

          # Install necessary dependencies
          apt-get update
          apt-get install -y curl sudo gnupg git wget

          # install go
          architecture=$(dpkg --print-architecture)
          wget -O go_tar.tar.gz https://go.dev/dl/go1.23.5.linux-${architecture}.tar.gz
          sudo tar -C /usr/local -xzf go_tar.tar.gz
          export PATH=${PATH}:/usr/local/go/bin

          # Clone and run the benchmark with read-stall-timeout set to true
          git clone -b read-stall-retry-scale-test https://github.com/raj-prince/custom-go-client-benchmark.git
          cd custom-go-client-benchmark
          go run ./ --worker=32 --bucket=vipin-us-west4-second --project=gcs-fuse-test --bucket-dir=1B --with-read-stall-timeout=true --target-percentile=0.99 --min-delay=1500ms --output-bucket-path vipin-logs-and-metrics/metrics/${JOB_NAME}/${POD_NAME}-metrics.csv --total-files-to-read=18000000 >> output-log.txt 2>&1
          echo "********* Run Complete ***********"

          # Set up Google Cloud SDK for future use (if needed)
          sudo echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] http://packages.cloud.google.com/apt cloud-sdk main" | sudo tee -a /etc/apt/sources.list.d/google-cloud-sdk.list
          sudo curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key --keyring /usr/share/keyrings/cloud.google.gpg add -
          apt-get update
          apt-get install -y google-cloud-sdk

          gcloud storage cp output-log.txt gs://vipin-logs-and-metrics/logs/${JOB_NAME}/${POD_NAME}-logs.txt
          echo "****** Upload Csv complete *******"