in api/v1/src/lib/appliers/bigQueryApplier.js [229:288]
async function applyPolicies(projectId, policyIds, fullRefresh) {
const labelKey = cfg.cdsManagedLabelKey;
let options = {};
const bigqueryUtil = new BigQueryUtil(projectId);
const bigQueryPermissionDiffProcedure = bigqueryUtil.getTableFqdn(projectId, cfg.cdsDatasetId, cfg.bigQueryPermissionDiffProcedureId);
if (!fullRefresh && policyIds && policyIds.length > 0) {
options = {
query: `CALL \`${bigQueryPermissionDiffProcedure}\`(@policyIds)`,
params: { policyIds: policyIds }
};
} else {
options = {
query: `CALL \`${bigQueryPermissionDiffProcedure}\`(null)`
};
}
const [rows] = await bigqueryUtil.executeQuery(options);
console.log(`BigQuery Permission Diff Result: ${JSON.stringify(rows, null, 3)}`);
if (fullRefresh === true) {
// Update all managed datasets and tables
const role = await runtimeConfig.bigQueryDataViewerRole(projectId);
const datasets = await bigqueryUtil.getDatasetsByLabel(labelKey, role);
for (const dataset of datasets) {
const datasetId = dataset.datasetId;
let dsPolicyRecord = underscore.findWhere(rows, { datasetId: datasetId, isTableBased: false });
let accounts = [];
if (dsPolicyRecord) {
accounts = dsPolicyRecord.accounts;
}
await performDatasetMetadataUpdate(projectId, datasetId, accounts);
const tables = await bigqueryUtil.getTablesByLabel(datasetId, labelKey);
for (const table of tables) {
const tableId = table.tableId;
let tbPolicyRecord = underscore.findWhere(rows, { datasetId: datasetId, tableId: tableId, isTableBased: true });
let tbAccounts = [];
if (tbPolicyRecord) {
tbAccounts = tbPolicyRecord.accounts;
}
await performTableMetadataUpdate(projectId, datasetId, tableId, tbAccounts);
}
}
} else {
// Differential update, iterate over result based on the policyId filter only
// No need to apply an additional filter.
for (const row of rows) {
const datasetId = row.datasetId;
const tableId = row.tableId;
if (row.isTableBased === true) {
console.log(`Iterating over table: ${datasetId}.${tableId}`);
await performTableMetadataUpdate(projectId, datasetId, tableId, row.accounts);
} else {
console.log(`Iterating over dataset: ${datasetId}`);
await performDatasetMetadataUpdate(projectId, datasetId, row.accounts);
}
}
}
}