in jobs/refreshQueryCache/task.ts [48:248]
async function refreshOrganization(
organizationIndex: number,
operations: Operations,
refreshSet: string,
queryCache: QueryCache,
organization: Organization): Promise<IRefreshOrganizationResults> {
const result: IRefreshOrganizationResults = {
organizationName: organization.name,
refreshSet: refreshSet,
started: new Date(),
finished: null,
consistencyStats: {
'delete': 0,
'new': 0,
'update': 0,
},
};
let organizationDetails = null;
try {
organizationDetails = await organization.getDetails();
} catch (organizationError) {
console.log(`Organization get details error: ${organizationError} for org ${organization.name}`);
console.dir(organizationError);
return;
}
const organizationId = organizationDetails.id.toString();
console.log(`refreshing ${organization.name} (id=${organizationId}) organization...`);
if (refreshSet === 'all' || refreshSet === 'organizations') {
try {
const organizationAdmins = await organization.getMembers({ ...slowRequestCacheOptions, role: OrganizationMembershipRoleQuery.Admin });
updateConsistencyStats(result.consistencyStats,
await cacheOrganizationMembers(queryCache, organizationId, organizationAdmins, OrganizationMembershipRole.Admin));
const memberIds = new Set<string>(organizationAdmins.map(admin => admin.id.toString()));
await sleep(sleepBetweenSteps);
const organizationMembers = await organization.getMembers({ ...slowRequestCacheOptions, role: OrganizationMembershipRoleQuery.Member });
console.log(`${organizationIndex}: organization ${organization.name} has ${organizationAdmins.length} admins and ${organizationMembers.length} members`);
updateConsistencyStats(result.consistencyStats,
await cacheOrganizationMembers(queryCache, organizationId, organizationMembers, OrganizationMembershipRole.Member));
organizationMembers.map(member => memberIds.add(member.id.toString()));
await sleep(sleepBetweenSteps);
// Cleanup any former members
const cachedMembers = await queryCache.organizationMembers(organizationId);
const potentialFormerMembers: IQueryCacheOrganizationMembership[] = [];
cachedMembers.map(cm => {
if (!memberIds.has(cm.userId)) {
potentialFormerMembers.push(cm);
}
});
updateConsistencyStats(result.consistencyStats,
await cleanupFormerMembers(operations, queryCache, organization, potentialFormerMembers));
} catch (orgMembersError) {
console.log(`refresh for organization ${organization.name} members was interrupted by an error`);
console.dir(orgMembersError);
}
}
if (refreshSet === 'all' || refreshSet === 'teams') {
try {
const teams = await organization.getTeams(slowRequestCacheOptions);
console.log(`${organizationIndex}: organization ${organization.name} has ${teams.length} teams`);
const knownTeams = new Set(teams.map(team => team.id.toString()));
await sleep(sleepBetweenSteps);
for (let i = 0; i < teams.length; i++) {
try {
const team = teams[i];
const teamDetailsData = await team.getDetails(minuteAgoCache);
updateConsistencyStats(result.consistencyStats,
await queryCache.addOrUpdateTeam(organizationId, team.id.toString(), teamDetailsData));
const teamMaintainers = await team.getMaintainers(slowRequestCacheOptions);
const maintainers = new Set<number>(teamMaintainers.map(maintainer => maintainer.id));
updateConsistencyStats(result.consistencyStats,
await cacheTeamMembers(queryCache, organizationId, team, teamMaintainers, GitHubTeamRole.Maintainer));
const teamMembers = await team.getMembers(slowRequestCacheOptions);
const knownTeamMembers = new Set(teamMembers.map(member => member.id.toString()));
const nonMaintainerMembers = teamMembers.filter(member => !maintainers.has(member.id));
updateConsistencyStats(result.consistencyStats,
await cacheTeamMembers(queryCache, organizationId, team, nonMaintainerMembers, GitHubTeamRole.Member));
console.log(`${organizationIndex}: team ${i + 1}/${teams.length}: ${team.name} from org ${organization.name} has ${teamMaintainers.length} maintainers and ${nonMaintainerMembers.length} members`);
// Cleanup any removed team members
const cachedTeamMembers = await queryCache.teamMembers(team.id.toString());
const removedMembers: IQueryCacheTeamMembership[] = [];
cachedTeamMembers.map(ctm => {
if (!knownTeamMembers.has(ctm.userId)) {
removedMembers.push(ctm);
}
});
updateConsistencyStats(result.consistencyStats,
await cleanupRemovedTeamMembers(queryCache, team, removedMembers));
await sleep(sleepBetweenSteps);
} catch (teamError) {
console.log(`issue processing team ${teams[i].id} in org ${organization.name}`);
console.dir(teamError);
await sleep(sleepBetweenSteps);
}
}
// Cleanup any removed teams
const cachedTeams = await queryCache.organizationTeams(organizationId);
const potentialFormerTeams: IQueryCacheTeam[] = [];
cachedTeams.map(ct => {
if (!knownTeams.has(ct.team.id.toString())) {
potentialFormerTeams.push(ct);
}
});
updateConsistencyStats(result.consistencyStats,
await cleanupFormerTeams(queryCache, organization, potentialFormerTeams));
} catch (refreshTeamsError) {
console.log(`error while refreshing teams in ${organization.name} org`);
console.dir(refreshTeamsError);
}
}
if (refreshSet === 'all' || refreshSet === 'collaborators' || refreshSet === 'permissions') {
const repositories = await organization.getRepositories(slowRequestCacheOptions);
console.log(`${organizationIndex}: ${repositories.length} repositories in ${organization.name}`);
const repoIds = new Set(repositories.map(repo => repo.id.toString()));
for (let i = 0; i < repositories.length; i++) {
try {
const repository = repositories[i];
const repoDetailsData = await repository.getDetails(minuteAgoCache);
updateConsistencyStats(result.consistencyStats,
await queryCache.addOrUpdateRepository(organizationId, repository.id.toString(), repoDetailsData));
await sleep(sleepBetweenSteps);
if (refreshSet === 'all' || refreshSet === 'permissions') {
const repoTeamPermissions = await repository.getTeamPermissions(slowRequestCacheOptions);
updateConsistencyStats(result.consistencyStats,
await cacheRepositoryTeams(queryCache, repository, repoTeamPermissions));
const knownTeamPermissions = new Set(repoTeamPermissions.map(rtp => rtp.team.id.toString()));
await sleep(sleepBetweenSteps);
// Cleanup any removed team permissions
const cachedTeamPermissions = await queryCache.repositoryTeamPermissions(repository.id.toString());
const removedPermissions: IQueryCacheTeamRepositoryPermission[] = [];
cachedTeamPermissions.map(ctp => {
if (!knownTeamPermissions.has(ctp.team.id.toString())) {
removedPermissions.push(ctp);
}
});
updateConsistencyStats(result.consistencyStats,
await cleanupRemovedTeamPermissions(queryCache, repository, removedPermissions));
}
if (refreshSet === 'all' || refreshSet === 'collaborators') {
const outsideOptions: IGetCollaboratorsOptions = { ...slowRequestCacheOptions, affiliation: GitHubCollaboratorAffiliationQuery.Outside };
const outsideRepoCollaborators = await repository.getCollaborators(outsideOptions);
const collaboratorIds = new Set(outsideRepoCollaborators.map(orc => orc.id.toString()));
updateConsistencyStats(result.consistencyStats,
await cacheRepositoryCollaborators(queryCache, organizationId, repository, outsideRepoCollaborators, GitHubCollaboratorType.Outside));
const outsideSet = new Set<number>(outsideRepoCollaborators.map(outsider => outsider.id));
const directOptions: IGetCollaboratorsOptions = { ...slowRequestCacheOptions, affiliation: GitHubCollaboratorAffiliationQuery.Direct };
const directRepoCollaborators = await repository.getCollaborators(directOptions);
directRepoCollaborators.map(drc => collaboratorIds.add(drc.id.toString()));
const insideDirectCollaborators = directRepoCollaborators.filter(collaborator => !outsideSet.has(collaborator.id));
// technically 'direct' is just those that are not outside collaborators
updateConsistencyStats(result.consistencyStats,
await cacheRepositoryCollaborators(queryCache, organizationId, repository, insideDirectCollaborators, GitHubCollaboratorType.Direct));
const cachedRepositoryCollaborators = await queryCache.repositoryCollaborators(repository.id.toString());
const formerCollaborators: IQueryCacheRepositoryCollaborator[] = [];
cachedRepositoryCollaborators.map(crc => {
if (!collaboratorIds.has(crc.userId)) {
formerCollaborators.push(crc);
}
});
updateConsistencyStats(result.consistencyStats,
await cleanupFormerCollaborators(queryCache, repository, formerCollaborators));
await sleep(sleepBetweenSteps);
}
console.log(`${organizationIndex}: repository ${i + 1}/${repositories.length}: ${repository.full_name} repository`);
} catch (refreshRepositoryError) {
console.dir(refreshRepositoryError);
await sleep(sleepBetweenSteps);
}
}
// Cleanup any deleted repos
try {
const cachedRepositories = await queryCache.organizationRepositories(organizationId);
const deletedRepositories: IQueryCacheRepository[] = [];
cachedRepositories.map(r => {
if (!repoIds.has(r.repository.id.toString())) {
deletedRepositories.push(r);
}
});
updateConsistencyStats(result.consistencyStats,
await cleanupDeletedRepositories(queryCache, organization, deletedRepositories));
} catch (cleanError) {
console.dir(cleanError);
await sleep(sleepBetweenSteps);
}
}
result.finished = new Date();
return result;
}