func calculateAndCheckWorkers()

in main.go [337:365]


func calculateAndCheckWorkers(reader *file.FileReader, size int64) {
	if workers > 0 {
		return
	}

	if !enableConcurrency {
		loadInfo.Workers = 1
		workers = 1
		return
	}

	ratio := float64(size) / float64(maxBytesPerTask)
	tmpWorkers := 0

	if ratio > 0.0 && ratio <= 0.001 {
		tmpWorkers = 1
	} else if ratio > 0.001 && ratio <= 0.01 {
		tmpWorkers = 2
	} else if ratio > 0.01 && ratio <= 0.1 {
		tmpWorkers = 4
	} else {
		tmpWorkers = 8
	}
	tmpWorkers = int(math.Min(float64(tmpWorkers), float64(diskThroughput)/float64(streamLoadThroughput)))

	log.Infof("worker number is %d, which is <= 0, trigger automatic inference. Final worker number is %d", workers, tmpWorkers)
	workers = tmpWorkers
	loadInfo.Workers = workers
}