in services/library/src/main/java/com/google/cloud/pso/bq_snapshot_manager/functions/f01_dispatcher/BigQueryScopeLister.java [160:206]
private List<TableSpec> processDatasets(List<String> datasetIncludeList,
List<String> datasetExcludeList,
List<String> tableExcludeList,
List<Pattern> datasetExcludeListPatterns,
List<Pattern> tableExcludeListPatterns
) {
List<String> tablesIncludeList = new ArrayList<>();
for (String dataset : datasetIncludeList) {
try {
Tuple<Boolean, String> checkResults = isIncluded(dataset, datasetExcludeList, datasetExcludeListPatterns);
if (!checkResults.x()) {
List<String> tokens = Utils.tokenize(dataset, ".", true);
String projectId = tokens.get(0);
String datasetId = tokens.get(1);
// get all tables under dataset
List<String> datasetTables = resourceScanner.listTables(projectId, datasetId);
tablesIncludeList.addAll(datasetTables);
if (datasetTables.isEmpty()) {
String msg = String.format(
"No Tables found under dataset '%s'",
dataset);
logger.logWarnWithTracker(runId, null, msg);
} else {
logger.logInfoWithTracker(runId, null, String.format("Found %s tables under dataset %s", datasetTables.size(), dataset));
}
} else {
logger.logInfoWithTracker(runId, null, String.format("Dataset %s is excluded by %s", dataset, checkResults.y()));
}
} catch (Exception exception) {
// log and continue
logger.logFailedDispatcherEntityId(runId, null, dataset, exception.getMessage(), exception.getClass().getName());
}
}
return processTables(
tablesIncludeList,
tableExcludeList,
tableExcludeListPatterns);
}