in iot-onboarding-service/src/cloudrack-lambda-core/s3/s3.go [423:467]
func (mpc S3MultipartConfig) Upload(fileName string) {
log.Printf("[Upload] Starting Upload of file %s", fileName)
//measuring time
start := time.Now()
//0-Opening File
file, err := os.Open(fileName)
if err != nil {
log.Printf("[Upload] error file opening the file: %+v", err)
os.Exit(1)
}
defer file.Close()
//0-Opening File
fileInfo, _ := file.Stat()
var fileSize int64 = fileInfo.Size()
log.Printf("[Upload] Starting Upload of file %s of size %v", fileName, fileSize)
fileChunk := mpc.ChunkSize * (1 << 20) // 1 MB, change this to your requirement
// calculate total number of parts the file will be chunked into
totalPartsNum := int64(math.Ceil(float64(fileSize) / float64(fileChunk)))
chunked := core.ChunkArray(totalPartsNum, int64(math.Ceil(float64(totalPartsNum)/float64(mpc.NThreads))))
log.Printf("[Upload] Splitting to %d pieces: %+v.\n", totalPartsNum, chunked)
var wg sync.WaitGroup
log.Printf("[Upload] Adding %+v thead to waitgroup.\n", len(chunked))
wg.Add(len(chunked))
for _, parts := range chunked {
//async
go func(parts []interface{}) {
for _, part := range parts {
partSize := int(math.Min(float64(fileChunk), float64(fileSize-int64(part.(int64)*fileChunk))))
log.Printf("[Upload] Uploading part %v of size %v bytes", part, partSize)
partBuffer := make([]byte, partSize)
file.Read(partBuffer)
//part number must be greater than 0
mpc.Send(part.(int64)+1, partBuffer)
log.Printf("[Upload] Uploading part %v Completed", part)
}
wg.Done()
}(parts)
}
wg.Wait()
elapsed := time.Since(start)
log.Printf("[Upload] Upload of file %s (size %v) completed in %s", fileName, fileSize, elapsed)
}