in services/library/src/main/java/com/google/cloud/pso/bq_snapshot_manager/functions/f01_dispatcher/BigQueryScopeLister.java [209:262]
private List<TableSpec> processProjects(
List<String> projectIncludeList,
List<String> projectExcludeList,
List<String> datasetExcludeList,
List<String> tableExcludeList,
List<Pattern> projectExcludeListPatterns,
List<Pattern> datasetExcludeListPatterns,
List<Pattern> tableExcludeListPatterns
) {
List<String> datasetIncludeList = new ArrayList<>();
logger.logInfoWithTracker(runId, null, String.format("Will process projects %s", projectIncludeList));
for (String project : projectIncludeList) {
try {
Tuple<Boolean, String> checkResults = isIncluded(project, projectExcludeList, projectExcludeListPatterns);
if (!checkResults.x()) {
logger.logInfoWithTracker(runId, null, String.format("Inspecting project %s", project));
// get all datasets in this project
List<String> projectDatasets = resourceScanner.listDatasets(project);
datasetIncludeList.addAll(projectDatasets);
if (projectDatasets.isEmpty()) {
String msg = String.format(
"No datasets found under project '%s' or no enough permissions to list BigQuery resources.",
project);
logger.logWarnWithTracker(runId, null, msg);
} else {
logger.logInfoWithTracker(runId, null, String.format("Datasets found in project %s : %s", project, projectDatasets));
}
} else {
logger.logInfoWithTracker(runId, null, String.format("Project %s is excluded by %s", project, checkResults.y()));
}
} catch (Exception exception) {
// log and continue
logger.logFailedDispatcherEntityId(runId, null, project, exception.getMessage(), exception.getClass().getName());
}
}
return processDatasets(
datasetIncludeList,
datasetExcludeList,
tableExcludeList,
datasetExcludeListPatterns,
tableExcludeListPatterns);
}