in odps-console-dship/src/main/java/com/aliyun/odps/ship/download/FileDownloader.java [102:162]
public void download() throws IOException, TunnelException {
if (sh != null) {
String msg = String.format("file [" + id + "] start");
sh.log(msg);
System.err.println(sim.format(new Date()) + " - " + msg);
}
String fd = DshipContext.INSTANCE.get(Constants.FIELD_DELIMITER);
String rd = DshipContext.INSTANCE.get(Constants.RECORD_DELIMITER);
String ni = DshipContext.INSTANCE.get(Constants.NULL_INDICATOR);
String dfp = DshipContext.INSTANCE.get(Constants.DATE_FORMAT_PATTERN);
String tz = DshipContext.INSTANCE.get(Constants.TIME_ZONE);
String charset = DshipContext.INSTANCE.get(Constants.CHARSET);
boolean exponential = false;
String e = DshipContext.INSTANCE.get(Constants.EXPONENTIAL);
if (e != null && e.equalsIgnoreCase("true")) {
exponential = true;
}
if (isCsv) {
writer = new CsvRecordWriter(file, charset);
} else {
writer = new TextRecordWriter(file, fd, rd);
}
RecordConverter converter = new RecordConverter(schema, ni, dfp, tz, charset, exponential, true);
if ("true".equalsIgnoreCase(DshipContext.INSTANCE.get(Constants.HEADER))) {
writeHeader(writer, schema);
}
preTime = System.currentTimeMillis();
DshipRecordReader recordReader = ds.getRecordReader(start, end);
long count = 0;
Record r;
while ((r = readAndTime(recordReader)) != null) {
writeAndTime(writer, converter.format(r));
count++;
currTime = System.currentTimeMillis();
// 5秒一次输出
if (currTime - preTime > 5000) {
printProgress(count);
preTime = currTime;
}
ODPSConsoleUtils.checkThreadInterrupted();
}
writer.close();
writtenBytes = writer.getWrittedBytes();
if (sh != null) {
StringBuilder messageBuilder = new StringBuilder();
messageBuilder.append(String.format("file [%d] OK. total: %s",
id,
Util.toReadableBytes(writtenBytes)));
messageBuilder.append(localIOStopWatch.getFormattedSummary());
messageBuilder.append(tunnelIOStopWatch.getFormattedSummary());
System.err.println(sim.format(new Date()) + " - " + messageBuilder.toString());
sh.log(messageBuilder.toString());
}
}