in tools/mc2bq/pkg/export/export.go [80:124]
func newExportTask(ctx context.Context, dataset *bigquery.Dataset, params *Params, src mcutil.ObjectSource, tableSuffix string, objectCount uint64) func() error {
tblName := params.TablePrefix + tableSuffix
return func() error {
done := make(chan bool, 1)
defer close(done)
go func() {
const timeout = 5 * time.Second
timer := time.NewTimer(timeout)
for {
select {
case <-done:
timer.Stop()
return
case <-timer.C:
if src.ObjectsRead() == objectCount || src.ObjectsRead() == 0 {
// Don't write progress if we haven't started or just finished
continue
}
fmt.Println(messages.ExportTableInProgress{
TableName: tblName,
RecordsTransferred: src.ObjectsRead(),
RecordCount: objectCount,
BytesTransferred: src.BytesRead(),
})
timer.Reset(timeout)
}
}
}()
err := exportObjects(ctx, dataset, params, src, tblName)
if err != nil {
return fmt.Errorf("export %s: %w", tableSuffix, err)
}
done <- true
fmt.Println(messages.ExportTableComplete{
TableName: tblName,
RecordCount: src.ObjectsRead(),
BytesTransferred: src.BytesRead(),
})
return nil
}
}