func processCUR()

in scripts/go/analyzeCUR/analyzeCUR.go [588:657]
54 lines of code
10 McCabe index (conditional complexity)

func processCUR(sourceBucket string, reportName string, reportPath string, destPath string, destBucket string, logger *cwlogger.Logger, dateOverride string) ([]curconvert.CurColumn, string, string, error) {

	var t1 time.Time
	var err error
	if len(dateOverride) == 8 {
		t1, err = time.Parse("20060102", dateOverride)
		if err != nil {
			return nil, "", "", errors.New("Could not parse given date ovrride: " + dateOverride + ", " + err.Error())
		}
	} else {
		t1 = time.Now()
	}

	t1First := time.Date(t1.Year(), t1.Month(), 1, 0, 0, 0, 0, time.Local)

	t2 := t1First.AddDate(0, 1, 0)
	t2First := time.Date(t2.Year(), t2.Month(), 1, 0, 0, 0, 0, time.Local)

	curDate := fmt.Sprintf("%d%02d01-%d%02d01", t1First.Year(), t1First.Month(), t2First.Year(), t2First.Month())
	manifest := reportPath + "/" + curDate + "/" + reportName + "-Manifest.json"

	// Set or extend destPath
	destPathDate := fmt.Sprintf("%d%02d", t1First.Year(), t1First.Month())
	var destPathFull string
	if len(destPath) < 1 {
		destPathFull = "parquet-cur/" + destPathDate
	} else {
		destPathFull = destPath + "/" + destPathDate
	}

	// Init CUR Converter
	cc := curconvert.NewCurConvert(sourceBucket, manifest, destBucket, destPathFull)

	// Check current months manifest exists
	if err := cc.CheckCURExists(); err != nil {
		if err.(awserr.Error).Code() != s3.ErrCodeNoSuchKey {
			return nil, "", "", errors.New("Error fetching CUR Manifest: " + err.Error())
		}
		if t1.Day() > 3 {
			return nil, "", "", errors.New("Error fetching CUR Manifest, NoSuchKey and too delayed: " + err.Error())
		}
		// Regress to processing last months CUR. Error is ErrCodeNoSuchKey and still early in the month
		doLog(logger, "Reseting to previous months CUR for "+reportName)
		t1First = t1First.AddDate(0, 0, -1)
		t2First = t2First.AddDate(0, 0, -1)
		curDate = fmt.Sprintf("%d%02d01-%d%02d01", t1First.Year(), t1First.Month(), t2First.Year(), t2First.Month())
		destPathDate = fmt.Sprintf("%d%02d", t1First.Year(), t1First.Month())

		manifest := reportPath + "/" + curDate + "/" + reportName + "-Manifest.json"
		cc.SetSourceManifest(manifest)

		if len(destPath) < 1 {
			destPathFull = "parquet-cur/" + destPathDate
		} else {
			destPathFull = destPath + "/" + destPathDate
		}
		cc.SetDestPath(destPathFull)
	}

	// Convert CUR
	if err := cc.ConvertCur(); err != nil {
		return nil, "", "", errors.New("Could not convert CUR: " + err.Error())
	}

	cols, err := cc.GetCURColumns()
	if err != nil {
		return nil, "", "", errors.New("Could not obtain CUR columns: " + err.Error())
	}
	return cols, "s3://" + destBucket + "/" + destPathFull + "/", destPathDate, nil
}