in workload-management/s3-trigger-ecs-task/s3-file-processor/main.go [16:63]
func main() {
inputFile, outputFile, err := initialize()
if err != nil {
failure("Pre-requisites failed unable to proceed further " + err.Error())
}
if inputFile != nil {
defer inputFile.Close()
}
if outputFile != nil {
defer outputFile.Close()
}
// Start parsing the inputFile
totalRecords := 0
reader := csv.NewReader(inputFile)
var records []*utils.Record
index := 0
// Skip first row header
record, errRead := reader.Read()
for {
record, errRead = reader.Read()
if errRead == io.EOF {
if len(records) > 0 {
processRecords(records, outputFile)
}
success()
break
}
if errRead != nil {
failure("Error while reading the inputFile records " + errRead.Error())
}
index++
totalRecords++
if index == utils.MaxRecordsPerBatch {
processRecords(records, outputFile)
// Trim data
records = records[:0]
index = 0
log.Printf("Total records so far - %v", totalRecords)
} else {
records = append(records, utils.MarshalRecord(record))
}
}
}