func main()

in workload-management/s3-trigger-ecs-task/s3-file-processor/main.go [16:63]


func main() {
	inputFile, outputFile, err := initialize()
	if err != nil {
		failure("Pre-requisites failed unable to proceed further " + err.Error())
	}
	if inputFile != nil {
		defer inputFile.Close()
	}
	if outputFile != nil {
		defer outputFile.Close()
	}

	// Start parsing the inputFile
	totalRecords := 0
	reader := csv.NewReader(inputFile)
	var records []*utils.Record
	index := 0

	// Skip first row header
	record, errRead := reader.Read()
	for {
		record, errRead = reader.Read()
		if errRead == io.EOF {
			if len(records) > 0 {
				processRecords(records, outputFile)
			}

			success()
			break
		}
		if errRead != nil {
			failure("Error while reading the inputFile records " + errRead.Error())
		}
		index++
		totalRecords++

		if index == utils.MaxRecordsPerBatch {
			processRecords(records, outputFile)

			// Trim data
			records = records[:0]
			index = 0
			log.Printf("Total records so far - %v", totalRecords)
		} else {
			records = append(records, utils.MarshalRecord(record))
		}
	}
}