in src/main/kotlin/org/jetbrains/teamcity/github/WebhookPeriodicalChecker.kt [119:297]
fun doCheck() {
LOG.info("Periodical GitHub Webhooks checker started")
val ignoredServers = ArrayList<String>()
myAuthDataCleaner.cleanup()
val toCheck = ArrayDeque(myWebHooksStorage.getAll())
val toPing = ArrayDeque<Triple<GitHubRepositoryInfo, Pair<GitHubClientEx, String>, WebHookInfo>>()
if (toCheck.isEmpty()) {
LOG.debug("No configured webhooks found")
} else {
LOG.debug("Will check ${toCheck.size} ${StringUtil.pluralize("webhook", toCheck.size)}")
}
while (toCheck.isNotEmpty()) {
val pair = toCheck.pop()
val (info, hook) = pair
val callbackUrl = hook.callbackUrl
val pubKey = GitHubWebHookListener.getPubKeyFromRequestPath(callbackUrl)
if (pubKey == null || pubKey.isBlank()) {
// Old hook format
LOG.warn("Callback url (${hook.callbackUrl}) of hook '${hook.url}' does not contains security check public key")
myWebHooksStorage.delete(hook)
continue
}
val authData = myAuthDataStorage.find(pubKey)
if (authData == null) {
if (TeamCityProperties.getBooleanOrTrue("teamcity.githubWebhooks.removeCorruptedHooks")) {
LOG.warn("Cannot find auth data for hook '${hook.url}', removing hook from storage")
myWebHooksStorage.delete(hook)
} else {
LOG.warn("Cannot find auth data for hook '${hook.url}'")
report(info, hook, "Webhook callback url is incorrect or internal storage was corrupted")
}
continue
}
val connectionInfo = authData.connection
val project = myProjectManager.findProjectByExternalId(connectionInfo.projectExternalId)
if (project == null) {
LOG.warn("OAuth Connection project '${connectionInfo.projectExternalId}' not found")
continue
}
val connection = myOAuthConnectionsManager.findConnectionById(project, connectionInfo.id)
if (connection == null) {
LOG.warn("OAuth Connection with id '${connectionInfo.id}' not found in project ${project.describe(true)} and it parents")
continue
}
val user = myUserModel.findUserById(authData.userId)
if (user == null) {
LOG.warn("TeamCity user '${authData.userId}' which created webhook for repository '${info.id}' no longer exists")
report(info, hook, "TeamCity user '${authData.userId}' which created webhook no longer exists", Status.NO_INFO)
continue
}
val tokens = myTokensHelper.getExistingTokens(project, listOf(connection), user).entries.firstOrNull()?.value.orEmpty()
if (tokens.isEmpty()) {
LOG.warn("No OAuth tokens to access repository '${info.id}'")
report(info, hook, "No OAuth tokens found to access repository", Status.NO_INFO)
continue
}
if (ignoredServers.contains(info.server)) {
// Server ignored for some time due to error on github
continue
}
val ghc = GitHubClientFactory.createGitHubClient(connection.parameters[GitHubConstants.GITHUB_URL_PARAM]!!)
var success = false
var retry = false
tokens@for (token in tokens) {
ghc.setOAuth2Token(token.accessToken)
try {
LOG.debug("Checking webhook status for '${info.id}' repository")
// GetAllWebHooksAction will automatically update statuses in all hooks for repository if succeed
val loaded = GetAllWebHooksAction.doRun(info, ghc, myWebHooksManager)
LOG.debug("Successfully fetched webhooks for '${info.id}' repository from GitHub server")
// Since we've loaded all hooks for repository 'info' it's safe to remove others for same repo from queue
toCheck.removeAll { it.first == info }
// Remove hooks removed on remote server from storages.
val removed = myWebHooksStorage.getHooks(info).filter { it.status == Status.MISSING }
if (removed.isNotEmpty()) {
LOG.info("$removed ${removed.size.pluralize("webhook")} missing on remote server and would be removed locally")
val pubKeysToRemove = removed.mapNotNull { GitHubWebHookListener.getPubKeyFromRequestPath(it.callbackUrl) }
myWebHooksStorage.delete(info) {it in removed}
myAuthDataStorage.remove(myAuthDataStorage.findAllForRepository(info).filter { it.public in pubKeysToRemove })
}
// Update info for all loaded hooks
for ((key, loadedHook) in loaded) {
val lastResponse = key.lastResponse
if (lastResponse == null || lastResponse.code == 0) {
LOG.debug("No last response info for hook ${key.url!!}")
// Lets ask GH to send us ping request, so next time there would be some 'lastResponse'
toPing.add(Triple(info, ghc to token.accessToken, loadedHook))
continue
}
when (lastResponse.code) {
in 200..299 -> {
LOG.debug("Last response is OK")
loadedHook.status = if (!key.isActive) Status.DISABLED else Status.OK
}
in 400..599 -> {
val reason = "Last payload delivery failed: (${lastResponse.code}) ${lastResponse.message}"
LOG.info(reason)
report(info, loadedHook, reason, Status.PAYLOAD_DELIVERY_FAILED)
}
else -> {
val reason = "Unexpected payload delivery response: (${lastResponse.code}) ${lastResponse.message}"
LOG.info(reason)
report(info, loadedHook, reason, Status.PAYLOAD_DELIVERY_FAILED)
}
}
}
success = true
break@tokens
} catch(e: GitHubAccessException) {
when (e.type) {
GitHubAccessException.Type.InvalidCredentials -> {
LOG.warn("Removing incorrect (outdated) token (user:${token.oauthLogin}, scope:${token.scope})")
myOAuthTokensStorage.removeToken(connection.tokenStorageId, token)
retry = true
}
GitHubAccessException.Type.TokenScopeMismatch -> {
LOG.warn("Token (user:${token.oauthLogin}, scope:${token.scope}) scope is not enough to check hook status")
myTokensHelper.markTokenIncorrect(token)
retry = true
}
GitHubAccessException.Type.UserHaveNoAccess -> {
LOG.warn("User (TC:${user.describe(false)}, GH:${token.oauthLogin}) have no access to repository ${info.id}, cannot check hook status")
if (tokens.map { it.oauthLogin }.distinct().size == 1) {
report(info, hook, "User (TC:${user.describe(false)}, GH:${token.oauthLogin}) installed webhook have no longer access to repository", Status.NO_INFO)
} else {
// TODO: ??? Seems TC user has many tokens with different GH users
}
retry = false
}
GitHubAccessException.Type.NoAccess -> {
LOG.warn("No access to repository ${info.id} for unknown reason, cannot check hook status")
retry = false
}
GitHubAccessException.Type.Moved -> {
LOG.info("Repository '${StringUtil.formatTextForWeb(info.id)}' was moved to ${e.message}")
retry = false
}
GitHubAccessException.Type.InternalServerError -> {
LOG.info("Cannot check hooks status for repository ${info.id}: Error on GitHub side. Will try later")
ignoredServers.add(info.server)
break@tokens
}
}
}
}
if (!success && retry) {
toCheck.add(pair)
}
checkQuotaLimit(ghc, ignoredServers, info)
}
for ((info, pair, hi) in toPing) {
if (ignoredServers.contains(info.server)) continue
val ghc = pair.first
ghc.setOAuth2Token(pair.second)
try {
TestWebHookAction.doRun(info, ghc, myWebHooksManager, hi)
} catch(e: GitHubAccessException) {
// Ignore
}
checkQuotaLimit(ghc, ignoredServers, info)
}
LOG.info("Periodical GitHub Webhooks checker finished")
}