in emr-dynamodb-hive/src/main/java/org/apache/hadoop/hive/dynamodb/DynamoDBSerDe.java [174:223]
private void verifyDynamoDBWriteThroughput(Configuration conf, Properties tbl) {
if (conf == null) {
// In a lot of places Hive creates a SerDe with null conf.
// In this case it is not possible to get the cluster status.
return;
}
if (warningPrinted) {
return;
}
String dynamoDBTableName = tbl.getProperty(DynamoDBConstants.TABLE_NAME);
// Hive uses partition metadata to initialize serde's. We may not need
// to verify write throughput at column level. dynamoDBTableName is null
// in this case, don't proceed and return
if (dynamoDBTableName == null) {
return;
}
log.info("Table Properties:" + tbl);
DynamoDBClient client = new DynamoDBClient(conf, tbl.getProperty(DynamoDBConstants.REGION));
long writesPerSecond = client.describeTable(dynamoDBTableName).getProvisionedThroughput()
.getWriteCapacityUnits();
long maxMapTasks;
try {
JobClient jc = new JobClient(new JobConf(conf));
maxMapTasks = jc.getClusterStatus().getMaxMapTasks();
} catch (IOException e) {
throw new RuntimeException("Could not get cluster capacity.", e);
}
BillingModeSummary billingModeSummary =
client.describeTable(dynamoDBTableName).getBillingModeSummary();
if (maxMapTasks > writesPerSecond
&& (billingModeSummary == null
|| billingModeSummary.getBillingMode().equals(
DynamoDBConstants.BILLING_MODE_PROVISIONED))) {
String message = "WARNING: Configured write throughput of the dynamodb table "
+ dynamoDBTableName + " is less than the cluster map capacity." + " ClusterMapCapacity: "
+ maxMapTasks + " WriteThroughput: " + writesPerSecond + "\nWARNING: Writes to this "
+ "table might result in a write outage on the table.";
LogHelper console = SessionState.getConsole();
if (console != null) {
console.printInfo(message);
}
log.warn(message);
warningPrinted = true;
}
}