private List processDatasets()

in services/library/src/main/java/com/google/cloud/pso/bq_snapshot_manager/functions/f01_dispatcher/BigQueryScopeLister.java [160:206]


    private List<TableSpec> processDatasets(List<String> datasetIncludeList,
                                            List<String> datasetExcludeList,
                                            List<String> tableExcludeList,
                                            List<Pattern> datasetExcludeListPatterns,
                                            List<Pattern> tableExcludeListPatterns
    ) {

        List<String> tablesIncludeList = new ArrayList<>();

        for (String dataset : datasetIncludeList) {

            try {

                Tuple<Boolean, String> checkResults = isIncluded(dataset, datasetExcludeList, datasetExcludeListPatterns);

                if (!checkResults.x()) {

                    List<String> tokens = Utils.tokenize(dataset, ".", true);
                    String projectId = tokens.get(0);
                    String datasetId = tokens.get(1);

                    // get all tables under dataset
                    List<String> datasetTables = resourceScanner.listTables(projectId, datasetId);
                    tablesIncludeList.addAll(datasetTables);

                    if (datasetTables.isEmpty()) {
                        String msg = String.format(
                                "No Tables found under dataset '%s'",
                                dataset);

                        logger.logWarnWithTracker(runId, null, msg);
                    } else {
                        logger.logInfoWithTracker(runId, null, String.format("Found %s tables under dataset %s", datasetTables.size(), dataset));
                    }
                } else {
                    logger.logInfoWithTracker(runId, null, String.format("Dataset %s is excluded by %s", dataset, checkResults.y()));
                }
            } catch (Exception exception) {
                // log and continue
                logger.logFailedDispatcherEntityId(runId, null, dataset, exception.getMessage(), exception.getClass().getName());
            }
        }
        return processTables(
                tablesIncludeList,
                tableExcludeList,
                tableExcludeListPatterns);
    }