in wayang-profiler/src/main/java/org/apache/wayang/profiler/java/Profiler.java [48:176]
public static void main(String[] args) {
if (args.length == 0) {
System.err.printf("Usage: java %s <operator to profile> <cardinality>[,<cardinality>]\n", Profiler.class);
System.exit(1);
}
String operator = args[0];
List<Integer> cardinalities = Arrays.stream(args[1].split(",")).map(Integer::valueOf).collect(Collectors.toList());
List<OperatorProfiler.Result> results;
switch (operator) {
case "textsource":
results = profile(OperatorProfilers.createJavaTextFileSourceProfiler(), cardinalities);
break;
case "collectionsource":
results = profile(OperatorProfilers.createJavaCollectionSourceProfiler(), cardinalities);
break;
case "map":
results = profile(OperatorProfilers.createJavaMapProfiler(), cardinalities);
break;
case "filter":
results = profile(OperatorProfilers.createJavaFilterProfiler(), cardinalities);
break;
case "flatmap":
results = profile(OperatorProfilers.createJavaFlatMapProfiler(), cardinalities);
break;
case "reduce":
results = profile(OperatorProfilers.createJavaReduceByProfiler(), cardinalities);
break;
case "globalreduce":
results = profile(OperatorProfilers.createJavaGlobalReduceProfiler(), cardinalities);
break;
case "distinct":
case "distinct-string":
results = profile(OperatorProfilers.createJavaDistinctProfiler(), cardinalities);
break;
case "distinct-integer":
results = profile(OperatorProfilers.createJavaDistinctProfiler(
DataGenerators.createReservoirBasedIntegerSupplier(new ArrayList<>(), 0.7d, new Random(42)),
Integer.class
), cardinalities);
break;
case "sort":
case "sort-string":
results = profile(OperatorProfilers.createJavaSortProfiler(), cardinalities);
break;
case "sort-integer":
results = profile(OperatorProfilers.createJavaSortProfiler(
DataGenerators.createReservoirBasedIntegerSupplier(new ArrayList<>(), 0.7d, new Random(42)),
Integer.class
), cardinalities);
break;
case "count":
results = profile(OperatorProfilers.createJavaCountProfiler(), cardinalities);
break;
case "groupby":
results = profile(OperatorProfilers.createJavaMaterializedGroupByProfiler(), cardinalities);
break;
case "join":
results = profile(OperatorProfilers.createJavaJoinProfiler(), cardinalities, cardinalities);
break;
case "union":
results = profile(OperatorProfilers.createJavaUnionProfiler(), cardinalities, cardinalities);
break;
case "cartesian":
results = profile(OperatorProfilers.createJavaCartesianProfiler(), cardinalities, cardinalities);
break;
case "callbacksink":
results = profile(OperatorProfilers.createJavaLocalCallbackSinkProfiler(), cardinalities);
break;
case "collect":
results = profile(OperatorProfilers.createCollectingJavaLocalCallbackSinkProfiler(), cardinalities);
break;
case "word-count-split": {
final Supplier<String> randomStringSupplier = DataGenerators.createRandomStringSupplier(2, 10, new Random(42));
results = profile(
OperatorProfilers.createJavaFlatMapProfiler(
() -> String.format("%s %s %s %s %s %s %s %s %s",
randomStringSupplier.get(), randomStringSupplier.get(),
randomStringSupplier.get(), randomStringSupplier.get(),
randomStringSupplier.get(), randomStringSupplier.get(),
randomStringSupplier.get(), randomStringSupplier.get(),
randomStringSupplier.get()),
str -> Arrays.asList(str.split(" ")),
String.class,
String.class
),
cardinalities);
break;
}
case "word-count-canonicalize": {
final Supplier<String> randomStringSupplier = DataGenerators.createRandomStringSupplier(2, 10, new Random(42));
results = profile(
OperatorProfilers.createJavaMapProfiler(
randomStringSupplier,
word -> new Tuple2<>(word.toLowerCase(), 1),
String.class,
Tuple2.class
),
cardinalities
);
break;
}
case "word-count-count": {
final Supplier<String> stringSupplier = DataGenerators.createReservoirBasedStringSupplier(new ArrayList<>(), 0.7, new Random(42), 2, 10);
results = profile(
OperatorProfilers.createJavaReduceByProfiler(
() -> new Tuple2<>(stringSupplier.get(), 1),
pair -> pair.field0,
(p1, p2) -> {
p1.field1 += p2.field1;
return p1;
},
cast(Tuple2.class),
String.class
),
cardinalities
);
break;
}
default:
System.out.println("Unknown operator: " + operator);
return;
}
System.out.println();
System.out.println(WayangCollections.getAny(results).getCsvHeader());
results.forEach(result -> System.out.println(result.toCsvString()));
}