update_deps/propagate_updates.ps1 (353 lines of code) (raw):
# Copyright (c) Microsoft. All rights reserved.
# Licensed under the MIT license. See LICENSE file in the project root for full license information.
<#
.SYNOPSIS
Propagates dependency updates for git repositories.
.DESCRIPTION
Given a root repo and personal access tokens for Github and Azure Devops Services, this script \
builds the dependency graph and propagates updates from the lowest level up to the \
root repo by making PRs to each repo in bottom-up level-order.
.PARAMETER root
Comma-separated list of URLs of the repositories upto which updates must be propagated.
.PARAMETER azure_token
Personal access token for Azure Devops Services
.PARAMETER azure_work_item
Work item id that is linked to PRs made to Azure repos.
.INPUTS
ignore.json: list of repositories that must be ignored for updates.
order.json: reads order in which repositories must be updated from order.json
.OUTPUTS
None.
.EXAMPLE
PS> .\{PATH_TO_SCRIPT}\propagate_updates.ps1 -azure_token {token1} -github_token {token2} -root_list root1, root2, ...
#>
param(
[Parameter(Mandatory=$true)][string]$azure_token, # Azure Devops Services personal access token: https://docs.microsoft.com/en-us/azure/devops/organizations/accounts/use-personal-access-tokens-to-authenticate?view=azure-devops&tabs=preview-page
[Parameter(Mandatory=$true)][Int32]$azure_work_item, # Azure Devops Services personal access token: https://docs.microsoft.com/en-us/azure/devops/organizations/accounts/use-personal-access-tokens-to-authenticate?view=azure-devops&tabs=preview-page
[Parameter(Mandatory=$true)][string[]]$root_list # comma-separated list of URLs for repositories upto which updates must be propagated
)
# sleep for $seconds seconds and play spinner animation
function spin {
param(
[int] $seconds
)
$steps = @('|','/','-','\')
$interval_ms = 50
for($i=0; $i -lt (($seconds*1000)/$interval_ms); $i++){
Write-Host "`b$($steps[$i % $steps.Length])" -NoNewline -ForegroundColor Yellow
Start-Sleep -Milliseconds $interval_ms
}
# erase spinner
Write-Host "`b"-NoNewLine
}
# create a global variable $ignore_pattern
# $ignore pattern is used in the shell command for 'git submodule foreach' to ignore repos
function create-ignore-pattern {
$path_to_ignores = $PSScriptRoot + "\ignores.json"
# get list of repos to ignore from ignores.json
$repos_to_ignore = (Get-Content -Path $path_to_ignores) | ConvertFrom-Json
$ignore_list = New-Object -TypeName "System.Collections.ArrayList"
# prepend "deps/" to the name of each repo
foreach($repo_to_ignore in $repos_to_ignore) {
[void]$ignore_list.Add("deps/"+$repo_to_ignore)
}
# join repo names to get pattern of the form "deps/{repo1}|deps/repo{2}|..."
$global:ignore_pattern = $ignore_list -join "|"
}
create-ignore-pattern
function refresh-submodules {
$submodules = git submodule | Out-String
Get-ChildItem "deps\" | ForEach-Object {
# There can be folders in deps\ that are not listed in .gitmodules.
# Only delete dep that is listed in .gitmodules
if($submodules.Contains($_.Name)) {
Remove-Item $_.FullName -Recurse -Force
}
}
}
# update the submodules of the given repo and push changes
# returns $true if the local repo was update
# returns $false if no changes were made
function update-local-repo {
param (
[string] $repo_name,
[string] $new_branch_name
)
cd $repo_name
git checkout master
git pull
# Sometimes git fails to detect updates in submodules
# Fix is to delete the submodule and reinitializes it
if (Test-Path "deps\") {
refresh-submodules
}
git submodule update --init
# update all submodules except the ones mentioned in ignores.json
git submodule foreach "case `$name in $ignore_pattern ) ;; *) git checkout master && git pull;; esac"
# create new branch
git checkout -B $new_branch_name
# add updates and push to remote
git add .
git commit -m "Update dependencies"
git push -f origin $new_branch_name
cd ..
}
function check-gh-cli-exists {
$gh = Get-Command gh -ErrorAction SilentlyContinue
if(!$gh) {
Write-Error "Github CLI is not installed. Install it from https://cli.github.com/"
exit -1
}
}
# update dependencies for Github repo
function update-repo-github {
param(
[string] $repo_name,
[string] $new_branch_name
)
cd $repo_name
Write-Host "`nCreating PR"
$working_directory = (Get-Location).Path
gh pr create --title "[autogenerated] update dependencies" --body "Propagating dependency updates" --head $new_branch_name
gh pr comment --body "/AzurePipelines run"
Write-Host "Waiting for checks to start"
Start-Sleep -Seconds 120
Write-Host "Waiting for build to complete"
gh pr checks --watch
Write-Host "Merging PR"
gh pr merge --squash --delete-branch
if($LASTEXITCODE -ne 0) {
Write-Error "Failed to merge PR for repo $repo_name"
exit -1
}
# Wait for merge to complete
Start-Sleep -Seconds 10
cd ..
}
# create global variable $azure_header
# $azure_header is used to authenticate requests to the Azure Devops Services API: https://docs.microsoft.com/en-us/rest/api/azure/devops/?view=azure-devops-rest-6.1
function create-header-azure {
param(
[string] $token
)
$base64_azure_pat = [Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes(":$token"))
$global:azure_header = @{
Authorization="Basic $base64_azure_pat"
}
}
create-header-azure $azure_token
# create PR to update dependencies for Azure repo
function create-pr-azure {
param(
[string] $repo_name,
[string] $new_branch_name
)
$request_url = "https://dev.azure.com/msazure/One/_apis/git/repositories/$repo_name/pullrequests?api-version=6.0"
$body = @{
sourceRefName="refs/heads/$new_branch_name"
targetRefName='refs/heads/master'
title='[autogenerated] update dependencies'
}
$body = $body | ConvertTo-Json
$response = Invoke-WebRequest -URI $request_url -UseBasicParsing -Method Post -Headers $azure_header -Body $body -ContentType "application/json"
if(!$response) {
Write-Error "Failed to create PR for repo $repo_name"
exit -1
}
$content = $response.Content | ConvertFrom-Json
return $content
}
# link work item to PR for Azure repo
function link-work-item-to-pr-azure {
param(
[string] $pr_artifact_id
)
if(!$azure_work_item){
Write-Error "Updating Azure repos requires providing a work item id. Provide work item id as: -azure_work_item [id]"
exit -1
}
$request_url = 'https://dev.azure.com/msazure/One/_apis/wit/workitems/'+$azure_work_item+'?api-version=6.0'
# body format found here: https://stackoverflow.com/questions/65111930/how-to-link-a-work-item-to-a-pull-request-using-rest-api-in-azure-devops
$body = @(
@{
op='add'
path='/relations/-'
value=@{
rel='ArtifactLink'
url=$pr_artifact_id
attributes=@{
name="pull request"
}
}
}
)
$body = $body | ConvertTo-Json
# This PATCH endpoint takes a list of patch objects so $body must be encased in a list
$response = Invoke-WebRequest -URI $request_url -UseBasicParsing -Method Patch -Headers $azure_header -Body "[$body]" -ContentType "application/json-patch+json"
if(!$response) {
Write-Error "Failed to link work item to PR.`nWork item: $work_item_id`nPR: $pr_artifact_id"
exit -1
}
}
# approve PR for Azure repo
function approve-pr-azure {
param(
[string] $pr_url,
[string] $creator_id
)
$request_url = $pr_url + '/reviewers/' + $creator_id + '?api-version=6.0'
$body = @{
vote=10
}
$body = $body | ConvertTo-Json
$response = Invoke-WebRequest -URI $request_url -UseBasicParsing -Method Put -Headers $azure_header -Body $body -ContentType "application/json"
if(!$response) {
Write-Error "Failed to approve PR: $pr_url"
exit -1
}
}
# set PR for Azure repo to merge automatically once build completes
function set-autocomplete-azure {
param(
$pr_url,
$creator_id
)
$request_url = $pr_url + '?api-version=6.0'
$body =@{
autoCompleteSetBy=@{
id=$creator_id
}
completionOptions=@{
mergeStrategy="squash"
deleteSourceBranch=$true
}
}
$body = $body | ConvertTo-Json
$response = Invoke-WebRequest -URI $request_url -UseBasicParsing -Method Patch -Headers $azure_header -Body $body -ContentType "application/json"
if(!$response) {
Write-Error "Failed to set autocomplete for PR $pr_url"
exit -1
}
}
# wait until build completes for Azure repo
function wait-until-complete-azure {
param(
[string] $pr_url
)
$status = ""
while($true){
$response = Invoke-WebRequest -URI $pr_url -UseBasicParsing -Method Get -Headers $azure_header
if(!$response) {
Write-Error "Failed to get PR: $pr_url"
exit -1
}
$content = $response.Content | ConvertFrom-Json
$status = $content.status
$merge_status = $content.mergeStatus
if($status -ne "active" -or !($merge_status -eq "succeeded" -or $merge_status -eq "queued")) {
break
}
spin 10
}
if($status -ne "completed") {
Write-Host "Problem with pull request: $pr_url"
Write-Error $response.Content
exit -1
}
}
# update dependencies for Azure repo
function update-repo-azure {
param(
[string] $repo_name,
[string] $new_branch_name
)
Write-Host "`nCreating PR"
$create_pr_response = create-pr-azure $repo_name $new_branch_name
$pr_artifact_id = $create_pr_response.artifactId
Write-Host "Linking work item to PR"
link-work-item-to-pr-azure $pr_artifact_id
Write-Host "Approving PR"
approve-pr-azure $create_pr_response.url $create_pr_response.createdBy.id
Write-Host "Enabling PR to autocomplete"
set-autocomplete-azure $create_pr_response.url $create_pr_response.createdBy.id
Write-Host "Waiting for build to complete"
wait-until-complete-azure $create_pr_response.url
}
# determine whether given repo is an azure repo or a github repo
function get-repo-type {
param (
[string] $repo_name
)
cd $repo_name
$repo_url = git config --get remote.origin.url
cd ..
Write-Host $repo_url -NoNewline
if($repo_url.Contains("github")){
return "github"
}elseif ($repo_url.Contains("azure")) {
return "azure"
}
return "unknown"
}
# update dependencies for given repo
function update-repo {
param(
[string] $repo_name,
[string] $new_branch_name
)
Write-Host "`n`nUpdating repo $repo_name"
[string]$git_output = (update-local-repo $repo_name $new_branch_name)
if($git_output.Contains("nothing to commit")) {
Write-Host "Nothing to commit, skipping repo $repo_name"
} else {
$repo_type = get-repo-type $repo_name
if($repo_type -eq "github") {
update-repo-github $repo_name $new_branch_name
} elseif ($repo_type -eq "azure") {
update-repo-azure $repo_name $new_branch_name
} else {
Write-Error "Unable to update repository $repo_name. Only Github and Azure repositories are supported."
exit -1
}
}
Write-Host "Done updating repo $repo_name"
}
function clear-directory {
$currentDirectory = Get-Location
$proceed = Read-Host("This script will clear the current directory ($currentDirectory). Enter [Y] to proceed.")
if($proceed -ne "Y")
{
exit 0
}
$Path = Get-Location | Select -expand Path
Set-Location ..
Remove-Item -LiteralPath $Path -Recurse -Force
$out = mkdir $Path
Set-Location $Path
}
# iterate over all repos and update them
function propagate-updates {
check-gh-cli-exists
clear-directory
# build dependency graph
Write-Host "Building dependency graph..."
.$PSScriptRoot\build_graph.ps1 -root_list $root_list
if($LASTEXITCODE -ne 0)
{
Write-Error("Could not build dependency graph for $root_list.")
exit -1
}
Write-Host "Done building dependency graph"
$repo_order = (Get-Content -Path order.json) | ConvertFrom-Json
Write-Host "Updating repositories in the following order: "
for($i = 0; $i -lt $repo_order.Length; $i++){
Write-Host "$($i+1). $($repo_order[$i])"
}
$new_branch_name = "new_deps_" + (Get-Date -Format "yyyyMMddHHmmss")
Write-Host "New branch name: $new_branch_name"
foreach ($repo in $repo_order) {
update-repo $repo $new_branch_name
}
Write-Host "Done updating all repos!"
}
propagate-updates