in SamplesV1/CustomizedJavaOnHDInsight/HDInsight-ADF/src/com/adf/jobonhdi/JobOnHdiLauncher.java [224:301]
private Set<String> downloadFiles(String filesToBeDownloaded, boolean isWindowsOs, String delim) throws Exception
{
Set<String> filesDownloaded = new HashSet<String>();
String[] files = filesToBeDownloaded.split(delim);
for (String path : files)
{
path = path.trim();
// Only copy Azure blob, skip copying from local worker nodes
if (path.startsWith("/"))
{
filesDownloaded.add(path);
continue;
}
if (path.toLowerCase().startsWith("wasb"))
{
String fileName = path.substring(path.lastIndexOf("/") + 1);
if (!filesDownloaded.contains(fileName))
{
HdfsCopy(path, fileName, isWindowsOs);
filesDownloaded.add(fileName);
}
continue;
}
Iterator<CloudBlobClient> it = blobClients.iterator();
int firstSlash = path.indexOf("/");
if (firstSlash == -1)
{
throw new RuntimeException("'" + path + "' is not a Azure storage blob in provided storage. Please use format container_name/blob_name.");
}
String fileName = path.substring(firstSlash + 1);
String destFileName = path.substring(path.lastIndexOf("/") + 1);
if (!filesDownloaded.contains(fileName))
{
String containerName = path.substring(0, firstSlash);
CloudBlobClient blobClient = null;
while (it.hasNext())
{
blobClient = it.next();
try
{
CloudBlobContainer container = blobClient.getContainerReference(containerName);
CloudBlockBlob blob = container.getBlockBlobReference(fileName);
blob.downloadToFile(destFileName);
filesDownloaded.add(destFileName);
System.out.println(path + " is copied successfully!");
break;
}
catch (Exception ex)
{
// ignore Exception
blobClient = null;
}
}
// Download fails
if (blobClient == null)
{
throw new RuntimeException("Failed to download '" + path + "'. Please provide credential of storage account.");
}
}
}
return filesDownloaded;
}