func()

in internal/scanners/diagnostics_settings.go [41:101]

• 43 lines of code
• 9 McCabe index (conditional complexity)

func (d *DiagnosticSettingsScanner) ListResourcesWithDiagnosticSettings(resources []*string) (map[string]bool, error) {
	res := map[string]bool{}

	if len(resources) > 5000 {
		log.Warn().Msg(fmt.Sprintf("%d resources detected. Scan will take longer than usual", len(resources)))
	}

	batches := int(math.Ceil(float64(len(resources)) / 20))

	models.LogResourceTypeScan("Diagnostic Settings")

	if batches == 0 {
		return res, nil
	}

	log.Debug().Msgf("Number of diagnostic setting batches: %d", batches)
	jobs := make(chan []*string, batches)
	ch := make(chan map[string]bool, batches)
	var wg sync.WaitGroup

	// Start workers
	// Based on: https://medium.com/insiderengineering/concurrent-http-requests-in-golang-best-practices-and-techniques-f667e5a19dea
	numWorkers := 100 // Define the number of workers in the pool
	for w := 0; w < numWorkers; w++ {
		go d.worker(jobs, ch, &wg)
	}
	wg.Add(batches)

	// Split resources into batches of 20 items.
	batchSize := 20
	batchCount := 0
	for i := 0; i < len(resources); i += batchSize {
		j := i + batchSize
		if j > len(resources) {
			j = len(resources)
		}
		jobs <- resources[i:j]

		batchCount++
		if batchCount == numWorkers {
			log.Debug().Msgf("all %d workers are running. Sleeping for 4 seconds to avoid throttling", numWorkers)
			batchCount = 0
			// there are more batches to process
			// Staggering queries to avoid throttling. Max 15 queries each 5 seconds.
			// https://learn.microsoft.com/en-us/azure/governance/resource-graph/concepts/guidance-for-throttled-requests#staggering-queries
			time.Sleep(4 * time.Second)
		}
	}

	// Wait for all workers to finish
	close(jobs)
	wg.Wait()

	for i := 0; i < batches; i++ {
		for k, v := range <-ch {
			res[k] = v
		}
	}

	return res, nil
}