in flink-runtime/src/main/java/org/apache/flink/runtime/operators/JoinDriver.java [90:275]
public void prepare() throws Exception {
final TaskConfig config = this.taskContext.getTaskConfig();
final Counter numRecordsIn =
this.taskContext.getMetricGroup().getIOMetricGroup().getNumRecordsInCounter();
// obtain task manager's memory manager and I/O manager
final MemoryManager memoryManager = this.taskContext.getMemoryManager();
final IOManager ioManager = this.taskContext.getIOManager();
// set up memory and I/O parameters
final double fractionAvailableMemory = config.getRelativeMemoryDriver();
final int numPages = memoryManager.computeNumberOfPages(fractionAvailableMemory);
// test minimum memory requirements
final DriverStrategy ls = config.getDriverStrategy();
final MutableObjectIterator<IT1> in1 =
new CountingMutableObjectIterator<>(
this.taskContext.<IT1>getInput(0), numRecordsIn);
final MutableObjectIterator<IT2> in2 =
new CountingMutableObjectIterator<>(
this.taskContext.<IT2>getInput(1), numRecordsIn);
// get the key positions and types
final TypeSerializer<IT1> serializer1 =
this.taskContext.<IT1>getInputSerializer(0).getSerializer();
final TypeSerializer<IT2> serializer2 =
this.taskContext.<IT2>getInputSerializer(1).getSerializer();
final TypeComparator<IT1> comparator1 = this.taskContext.getDriverComparator(0);
final TypeComparator<IT2> comparator2 = this.taskContext.getDriverComparator(1);
final TypePairComparatorFactory<IT1, IT2> pairComparatorFactory =
config.getPairComparatorFactory(this.taskContext.getUserCodeClassLoader());
if (pairComparatorFactory == null) {
throw new Exception("Missing pair comparator factory for join driver");
}
ExecutionConfig executionConfig = taskContext.getExecutionConfig();
boolean objectReuseEnabled = executionConfig.isObjectReuseEnabled();
if (LOG.isDebugEnabled()) {
LOG.debug(
"Join Driver object reuse: "
+ (objectReuseEnabled ? "ENABLED" : "DISABLED")
+ ".");
}
boolean hashJoinUseBitMaps =
taskContext
.getTaskManagerInfo()
.getConfiguration()
.get(AlgorithmOptions.HASH_JOIN_BLOOM_FILTERS);
// create and return joining iterator according to provided local strategy.
if (objectReuseEnabled) {
switch (ls) {
case INNER_MERGE:
this.joinIterator =
new ReusingMergeInnerJoinIterator<>(
in1,
in2,
serializer1,
comparator1,
serializer2,
comparator2,
pairComparatorFactory.createComparator12(
comparator1, comparator2),
memoryManager,
ioManager,
numPages,
this.taskContext.getContainingTask());
break;
case HYBRIDHASH_BUILD_FIRST:
this.joinIterator =
new ReusingBuildFirstHashJoinIterator<>(
in1,
in2,
serializer1,
comparator1,
serializer2,
comparator2,
pairComparatorFactory.createComparator21(
comparator1, comparator2),
memoryManager,
ioManager,
this.taskContext.getContainingTask(),
fractionAvailableMemory,
false,
false,
hashJoinUseBitMaps);
break;
case HYBRIDHASH_BUILD_SECOND:
this.joinIterator =
new ReusingBuildSecondHashJoinIterator<>(
in1,
in2,
serializer1,
comparator1,
serializer2,
comparator2,
pairComparatorFactory.createComparator12(
comparator1, comparator2),
memoryManager,
ioManager,
this.taskContext.getContainingTask(),
fractionAvailableMemory,
false,
false,
hashJoinUseBitMaps);
break;
default:
throw new Exception(
"Unsupported driver strategy for join driver: " + ls.name());
}
} else {
switch (ls) {
case INNER_MERGE:
this.joinIterator =
new NonReusingMergeInnerJoinIterator<>(
in1,
in2,
serializer1,
comparator1,
serializer2,
comparator2,
pairComparatorFactory.createComparator12(
comparator1, comparator2),
memoryManager,
ioManager,
numPages,
this.taskContext.getContainingTask());
break;
case HYBRIDHASH_BUILD_FIRST:
this.joinIterator =
new NonReusingBuildFirstHashJoinIterator<>(
in1,
in2,
serializer1,
comparator1,
serializer2,
comparator2,
pairComparatorFactory.createComparator21(
comparator1, comparator2),
memoryManager,
ioManager,
this.taskContext.getContainingTask(),
fractionAvailableMemory,
false,
false,
hashJoinUseBitMaps);
break;
case HYBRIDHASH_BUILD_SECOND:
this.joinIterator =
new NonReusingBuildSecondHashJoinIterator<>(
in1,
in2,
serializer1,
comparator1,
serializer2,
comparator2,
pairComparatorFactory.createComparator12(
comparator1, comparator2),
memoryManager,
ioManager,
this.taskContext.getContainingTask(),
fractionAvailableMemory,
false,
false,
hashJoinUseBitMaps);
break;
default:
throw new Exception(
"Unsupported driver strategy for join driver: " + ls.name());
}
}
// open the iterator - this triggers the sorting or hash-table building
// and blocks until the iterator is ready
this.joinIterator.open();
if (LOG.isDebugEnabled()) {
LOG.debug(this.taskContext.formatLogString("join task iterator ready."));
}
}