in metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/bulk/HDFSDataPruner.java [58:140]
public static void main(String... argv) throws IOException, java.text.ParseException, ClassNotFoundException, InterruptedException {
/**
* Example
* start=$(date -d '30 days ago' +%m/%d/%Y)
* yarn jar Metron-DataLoads-0.1BETA.jar org.apache.metron.dataloads.bulk.HDFSDataPruner -f hdfs://ec2-52-36-25-217.us-west-2.compute.amazonaws.com:8020 -g '/apps/metron/enrichment/indexed/bro_doc/*enrichment-*' -s $(date -d '30 days ago' +%m/%d/%Y) -n 1;
* echo ${start}
**/
Options options = new Options();
Options help = new Options();
{
Option o = new Option("h", "help", false, "This screen");
o.setRequired(false);
help.addOption(o);
}
{
Option o = new Option("s", "start-date", true, "Starting Date (MM/DD/YYYY)");
o.setArgName("START_DATE");
o.setRequired(true);
options.addOption(o);
}
{
Option o = new Option("f", "filesystem", true, "Filesystem uri - e.g. hdfs://host:8020 or file:///");
o.setArgName("FILESYSTEM");
o.setRequired(true);
options.addOption(o);
}
{
Option o = new Option("n", "numdays", true, "Number of days back to purge");
o.setArgName("NUMDAYS");
o.setRequired(true);
options.addOption(o);
}
{
Option o = new Option("g", "glob-string", true, "Glob filemask for files to delete - e.g. /apps/metron/enrichment/bro_doc/file-*");
o.setArgName("GLOBSTRING");
o.setRequired(true);
options.addOption(o);
}
try {
CommandLineParser parser = new PosixParser();
CommandLine cmd = null;
try {
cmd = parser.parse(help,argv,true);
if( cmd.getOptions().length > 0){
final HelpFormatter usageFormatter = new HelpFormatter();
usageFormatter.printHelp("HDFSDataPruner", null, options, null, true);
System.exit(0);
}
cmd = parser.parse(options, argv);
} catch (ParseException pe) {
final HelpFormatter usageFormatter = new HelpFormatter();
usageFormatter.printHelp("HDFSDataPruner", null, options, null, true);
System.exit(-1);
}
String start = cmd.getOptionValue("s");
Date startDate = new SimpleDateFormat("MM/dd/yyyy").parse(start);
String fileSystemUri = cmd.getOptionValue("f");
Integer numDays = Integer.parseInt(cmd.getOptionValue("n"));
String globString = cmd.getOptionValue("g");
LOG.debug("Running prune with args: {} {} {} {}", startDate, numDays, fileSystemUri, globString);
DataPruner pruner = new HDFSDataPruner(startDate, numDays, fileSystemUri, globString);
LOG.info("Pruned {} files from {}{}", pruner.prune(), fileSystemUri, globString);
} catch (Exception e) {
e.printStackTrace();
System.exit(-1);
}
}