in odps-sdk-impl/odps-common-local/src/main/java/com/aliyun/odps/local/common/utils/DownloadUtils.java [245:353]
public static void downloadResource(Odps odps, String projName, String resourceName,
int limitDownloadRecordCount, char inputColumnSeperator) {
if (odps == null || StringUtils.isBlank(projName) || StringUtils.isBlank(resourceName)) {
throw new IllegalArgumentException("Missing arguments: odps|projName|resourceName");
}
if (WareHouse.getInstance().getDownloadMode() == DownloadMode.NEVER) {
throw new RuntimeException("Download resource '" + projName + "." + resourceName
+ "' Failed! Current download mode is:" + DownloadMode.NEVER + ".Please check parameter'"
+ Constants.LOCAL_DOWNLOAD_MODE + "'");
}
try {
if (!odps.resources().exists(projName, resourceName)) {
throw new RuntimeException("Download resource: " + projName + "." + resourceName
+ " Failed!Remote resource not found! Download mode:"
+ WareHouse.getInstance().getDownloadMode());
}
} catch (OdpsException e1) {
throw new RuntimeException(e1);
}
Resource resource = odps.resources().get(projName, resourceName);
File resFile = WareHouse.getInstance().getReourceFile(projName, resourceName);
LOG.info("Start to download resource: " + resource.getName() + "-->"
+ resFile.getAbsolutePath() + ", type: " + resource.getType() + ",download mode:"
+ WareHouse.getInstance().getDownloadMode());
switch (resource.getType()) {
case FILE:
case JAR:
case PY:
case ARCHIVE:
InputStream in = null;
FileOutputStream out = null;
try {
in = odps.resources().getResourceAsStream(resource.getProject(), resource.getName());
resFile.getParentFile().mkdirs();
out = new FileOutputStream(resFile);
IOUtils.copy(in, out);
} catch (IOException e) {
throw new RuntimeException(e);
} catch (OdpsException e) {
throw new RuntimeException(e);
} finally {
if (in != null) {
try {
in.close();
} catch (IOException e) {
throw new RuntimeException(e);
}
}
if (out != null) {
try {
out.close();
} catch (IOException e) {
throw new RuntimeException(e);
}
}
}
break;
case TABLE:
TableResource tr = new TableResource(resource);
Table stable = tr.getSourceTable();
String sourceProjName = stable.getProject();
String sourceTbleName = stable.getName();
PartitionSpec partSpec = tr.getSourceTablePartition();
// 下载数据table到warehouse/_table__
TableInfo tableInfo = null;
if (partSpec == null) {
tableInfo = TableInfo.builder().projectName(sourceProjName).tableName(sourceTbleName)
.build();
// 下载表所有分区数据(每张表最多limit条数)
if (!WareHouse.getInstance().existsTable(sourceProjName, sourceTbleName)) {
downloadTableSchemeAndData(odps, tableInfo, limitDownloadRecordCount,
inputColumnSeperator);
}
} else {
LinkedHashMap<String, String> partMap = PartitionUtils.convert(partSpec);
tableInfo = TableInfo.builder().projectName(sourceProjName).tableName(sourceTbleName)
.partSpec(partMap).build();
// 下载分区数据
if (!WareHouse.getInstance().existsPartition(sourceProjName, sourceTbleName, partSpec)) {
downloadTableSchemeAndData(odps, tableInfo, limitDownloadRecordCount,
inputColumnSeperator);
}
}
// 创建表资源目录warehouse/__resource__/table_resource_name
WareHouse.getInstance().createTableReourceFile(resource.getProject(), resource.getName(),
tableInfo);
break;
default:
break;
}
LOG.info("Finished download resource: " + resource.getName() + "-->"
+ resFile.getAbsolutePath() + ", type: " + resource.getType() + ",download mode:"
+ WareHouse.getInstance().getDownloadMode());
}