in src/main/java/com/google/cloud/solutions/autotokenize/dlp/BatchAndDlpDeIdRecords.java [285:316]
public void processBatch(
@Element KV<ShardedKey<String>, PartialColumnDlpTable> batchKv,
ProcessContext processContext) {
var batch = batchKv.getValue();
var batchTable = batch.getTable();
logger.atFine().log(
"Sending Batch:%nBytes:%s%nColumns:%s%nRowCount:%s",
batchTable.getSerializedSize(),
DeidentifyColumns.columnNamesFromHeaders(batchTable.getHeadersList()),
batchTable.getRowsCount());
try {
DeidentifyContentResponse deidResponse =
dlpClient.deidentifyContent(
DeidentifyContentRequest.newBuilder()
.setParent(DlpUtil.makeDlpParent(dlpProjectId(), dlpRegion()))
.setDeidentifyConfig(batch.getDeidentifyConfig())
.setItem(ContentItem.newBuilder().setTable(batchTable).build())
.build());
processContext.output(
successTag(),
TokenizedDataMerger.create(batch, deidResponse.getItem().getTable(), encryptConfig())
.merge());
} catch (RuntimeException exception) {
logger.atSevere().withCause(exception).log("ErrorProcessing batch");
processContext.output(errorTag(), batchKv);
}
}