eng/scripts/Create-APIView.ps1 (531 lines of code) (raw):
. $PSScriptRoot/ChangedFiles-Functions.ps1
. $PSScriptRoot/../common/scripts/logging.ps1
$defaultTagRegex = "^tag:\s*(?<tag>.+)"
$tagRegex = '^```\s*yaml\s*\$\(tag\)\s*==\s*''(?<tag>.+)'''
<#
.DESCRIPTION
Gets configuration info (tags and configFilePath) associated with a swagger file.
.PARAMETER SwaggerFile
Path to a swagger files inside the 'specification' directory.
.OUTPUTS
the configuration info (tags and configFilePath) or null if not found.
#>
function Get-AutoRestConfigInfo {
param (
[Parameter(Mandatory = $true)]
[string]$SwaggerFile
)
$currentPath = Resolve-Path $SwaggerFile
while ($currentPath -ne [System.IO.Path]::GetPathRoot($currentPath)) {
$currentPath = [System.IO.Path]::GetDirectoryName($currentPath)
$currentFilePath = [System.IO.Path]::GetFileName($currentPath)
if ($currentFilePath -eq "specification") {
break
}
$readmeFile = Get-ChildItem -Path $currentPath -Filter "readme.md" -File -ErrorAction SilentlyContinue
if ($readmeFile -and $readmeFile.Name -eq "readme.md") {
$tagInfo = Get-TagInformationFromReadMeFile -ReadMeFilePath $readmeFile.FullName
if ($tagInfo.DefaultTag) {
$tagInfo | Add-Member -MemberType NoteProperty -Name "ConfigPath" -Value $readmeFile
return $tagInfo
}
}
}
return $null
}
<#
.DESCRIPTION
Use the directory structure convention to get the resource provider name.
Append the service directory name if there are multiple services in the same resource provider
.PARAMETER ReadMeFilePath
ReadMe File Path for a resource provider.
.OUTPUTS
The resource provider name.
#>
function Get-ResourceProviderFromReadMePath {
param (
[Parameter(Mandatory = $true)]
[string]$ReadMeFilePath
)
$directoryPath = [System.IO.Path]::GetDirectoryName($ReadMeFilePath)
$pathName = [System.IO.Path]::GetFileName($directoryPath)
if ($pathName -eq "resource-manager" -or $pathName -eq "data-plane") {
$resourceProviderDirectory = Get-ChildItem -Path $directoryPath -Directory | Select-Object -First 1
return $resourceProviderDirectory.Name
}
else {
$currentPath = Resolve-Path $directoryPath
$serviceName = $pathName
while ($currentPath -ne [System.IO.Path]::GetPathRoot($currentPath)) {
$pathName = [System.IO.Path]::GetFileName($currentPath)
if ($pathName -eq "resource-manager" -or $pathName -eq "data-plane") {
$resourceProviderDirectory = Get-ChildItem -Path $currentPath -Directory | Select-Object -First 1
return $resourceProviderDirectory.Name + "-" + $serviceName
}
$currentPath = Resolve-Path ([System.IO.Path]::GetDirectoryName($currentPath))
}
}
return $null
}
function Get-ImpactedTypespecProjects {
param (
[Parameter(Mandatory = $true)]
[string]$TypeSpecFile
)
$it = $TypeSpecFile
while ($it -and !$configFilesInTypeSpecProjects) {
$it = Split-Path -Parent $it
$configFilesInTypeSpecProjects = Get-ChildItem -Path $it -File "tspconfig.yaml"
}
if ($configFilesInTypeSpecProjects) {
foreach($configFilesInTypeSpecProject in $configFilesInTypeSpecProjects) {
$entryPointFile = Get-ChildItem -Path $($configFilesInTypeSpecProject.Directory.FullName) -File "main.tsp"
if ($entryPointFile) {
Write-Host "Found $($configFilesInTypeSpecProject.Name) and $($entryPointFile.Name) in directory $($configFilesInTypeSpecProject.Directory.FullName)"
return $configFilesInTypeSpecProject.Directory.FullName
}
else {
Write-Host "Did not find main.tsp in directory $($configFilesInTypeSpecProject.Directory.FullName)"
}
}
}
}
<#
.DESCRIPTION
Invoke the swagger parset to generate APIView tokens.
.PARAMETER Type
New or Baseline swagger APIView tokens.
.PARAMETER ReadMeFilePath
The Swagger ReadMeFilePath.
.PARAMETER ResourceProvider
The ResourceProvider Name.
.PARAMETER Tag
The Tag to use for generating the APIView Tokens.
.PARAMETER TokenDirectory
The directory to store the generated APIView Tokens.
.OUTPUTS
The resource provider name.
#>
function Invoke-SwaggerAPIViewParser {
param (
[ValidateSet("New", "Baseline")]
[Parameter(Mandatory = $true)]
[string]$Type,
[Parameter(Mandatory = $true)]
[string]$ReadMeFilePath,
[Parameter(Mandatory = $true)]
[string]$ResourceProvider,
[Parameter(Mandatory = $true)]
[string]$TokenDirectory,
[Parameter(Mandatory = $true)]
[string]$TempDirectory,
[string]$Tag
)
$tempWorkingDirectoryName = [guid]::NewGuid().ToString()
$tempWorkingDirectoryPath = [System.IO.Path]::Combine($TempDirectory, $tempWorkingDirectoryName)
New-Item -ItemType Directory -Path $tempWorkingDirectoryPath > $null
Push-Location -Path $tempWorkingDirectoryPath
try {
# Generate Swagger APIView tokens
$command = "swaggerAPIParser"
$arguments = @("--readme", "$ReadMeFilePath", "--package-name", "$ResourceProvider")
if ($Tag) {
$arguments += "--tag"
$arguments += "$Tag"
}
LogInfo " $command $arguments"
LogGroupStart " Generating '$Type' APIView Tokens using '$ReadMeFilePath' for '$ResourceProvider'..."
& $command @arguments 2>&1 | ForEach-Object { Write-Host $_ }
LogGroupEnd
$generatedAPIViewTokenFile = Join-Path (Get-Location) "swagger.json"
if (Test-Path -Path $generatedAPIViewTokenFile) {
LogSuccess " Generated '$Type' APIView Token File using file, '$ReadMeFilePath' and tag '$Tag'"
$apiViewTokensFilePath = [System.IO.Path]::Combine($TokenDirectory, "$ResourceProvider.$Type.json")
LogInfo " Moving generated APIView Token file to '$apiViewTokensFilePath'"
Move-Item -Path $generatedAPIViewTokenFile -Destination $apiViewTokensFilePath -Force > $null
}
} catch {
LogError " Failed to generate '$Type' APIView Tokens using '$ReadMeFilePath' for '$ResourceProvider'"
throw
} finally {
Pop-Location
if (Test-Path -Path $tempWorkingDirectoryPath) {
Remove-Item -Path $tempWorkingDirectoryPath -Recurse -Force > $null
}
}
}
<#
.DESCRIPTION
Invoke the TypeSpec parser to generate APIView tokens.
.PARAMETER Type
New or Baseline TypeSpec APIView tokens.
.PARAMETER ProjectPath
The TypeSpec Project path.
.PARAMETER ResourceProvider
The ResourceProvider Name.
.PARAMETER Tag
The Tag to use for generating the APIView Tokens.
.PARAMETER TokenDirectory
The directory to store the generated APIView Tokens.
.OUTPUTS
The resource provider name.
#>
function Invoke-TypeSpecAPIViewParser {
param (
[ValidateSet("New", "Baseline")]
[Parameter(Mandatory = $true)]
[string]$Type,
[Parameter(Mandatory = $true)]
[string]$ProjectPath,
[Parameter(Mandatory = $true)]
[string]$ResourceProvider,
[Parameter(Mandatory = $true)]
[string]$TokenDirectory
)
$tempWorkingDirectoryName = [guid]::NewGuid().ToString()
$tempWorkingDirectoryPath = [System.IO.Path]::Combine($TempDirectory, $tempWorkingDirectoryName)
New-Item -ItemType Directory -Path $tempWorkingDirectoryPath > $null
try {
Write-Host "Compiling files and generating '$Type' APIView for '$resourceProvider'..."
Push-Location $ProjectPath
Write-Host "npm exec --no -- tsp compile . --emit=@azure-tools/typespec-apiview --option @azure-tools/typespec-apiview.emitter-output-dir=$tempWorkingDirectoryPath/output/apiview.json"
npm exec --no -- tsp compile . --emit=@azure-tools/typespec-apiview --option @azure-tools/typespec-apiview.emitter-output-dir=$tempWorkingDirectoryPath/output/apiview.json
if ($LASTEXITCODE) {
throw "Compilation error when running: 'npm exec --no -- tsp compile . --emit=@azure-tools/typespec-apiview --option @azure-tools/typespec-apiview.emitter-output-dir=$tempWorkingDirectoryPath/output/apiview.json'"
}
Pop-Location
$generatedAPIViewTokenFile = Get-ChildItem -File $tempWorkingDirectoryPath/output/apiview.json | Select-Object -First 1
$apiViewTokensFilePath = [System.IO.Path]::Combine($TokenDirectory, "$resourceProvider.$Type.json")
Write-Host "Moving generated APIView Token file to '$apiViewTokensFilePath'"
Move-Item -Path $generatedAPIViewTokenFile.FullName -Destination $apiViewTokensFilePath -Force > $null
} catch {
LogError " Failed to generate '$Type' APIView Tokens on '$ProjectPath' for '$resourceProvider', please check the detail log and make sure TypeSpec compiler version is the latest."
LogError $_
throw
} finally {
if (Test-Path -Path $tempWorkingDirectoryPath) {
Remove-Item -Path $tempWorkingDirectoryPath -Recurse -Force > $null
}
}
}
<#
.DESCRIPTION
Generate New and Baseline APIView tokens for the changed swagger files in the PR.
Detects the swagger files changed in the PR and generates APIView tokens for the swagger files.
New APIView tokens are generated using the default tag on the base branch.
Baseline APIView tokens are generated using the same tag on the target branch.
Script asumes that the merge commit is checked out. Such that Source commit = HEAD^ and Target commit = HEAD.
.PARAMETER TempDirectory
Temporary directory for files being processed. Use $(Agent.TempDirectory) on DevOps
.PARAMETER ArtifactsStagingDirectory
The directory where the APIView tokens will be stored. Use $(Build.ArtifactStagingDirectory) on DevOps
.PARAMETER APIViewArtifactsDirectoryName
Name for the subdirectory where the APIView tokens will be stored.
#>
function New-SwaggerAPIViewTokens {
param (
[Parameter(Mandatory = $true)]
[string]$TempDirectory,
[Parameter(Mandatory = $true)]
[string]$ArtifactsStagingDirectory,
[Parameter(Mandatory = $true)]
[string]$APIViewArtifactsDirectoryName
)
$SourceCommitId = $(git rev-parse HEAD^2)
$TargetCommitId = $(git rev-parse HEAD^1)
# Get Changed Swagger Files
LogInfo " Getting changed swagger files in PR, between $SourceCommitId and $TargetCommitId"
$changedFiles = Get-ChangedFiles
$changedSwaggerFiles = Get-ChangedSwaggerFiles -changedFiles $changedFiles
if ($changedSwaggerFiles.Count -eq 0) {
LogWarning " There are no changes to swagger files in the current PR..."
Write-Host "##vso[task.complete result=SucceededWithIssues;]DONE"
exit 0
}
LogGroupStart " Pullrequest has changes in these swagger files..."
$changedSwaggerFiles | ForEach-Object {
LogInfo " - $_"
}
LogGroupEnd
# Get Related AutoRest Configuration Information
$autoRestConfigInfo = [System.Collections.Generic.Dictionary[string, object]]::new()
$changedSwaggerFiles | ForEach-Object {
$configInfo = Get-AutoRestConfigInfo -swaggerFile $_
if ($null -ne $configInfo -and -not $autoRestConfigInfo.ContainsKey($configInfo.ConfigPath)) {
$autoRestConfigInfo[$configInfo.ConfigPath] = $configInfo
}
}
if ($autoRestConfigInfo.Count -eq 0) {
LogWarning " No AutoRest configuration found for the changed swagger files in the current PR..."
Write-Host "##vso[task.complete result=SucceededWithIssues;]DONE"
exit 0
}
LogGroupStart " Swagger APIView Tokens will be generated for the following configuration files..."
$autoRestConfigInfo.GetEnumerator() | ForEach-Object {
LogInfo " - $($_.Key)"
}
LogGroupEnd
$currentBranch = git rev-parse --abbrev-ref HEAD
$swaggerAPIViewArtifactsDirectory = [System.IO.Path]::Combine($ArtifactsStagingDirectory, $APIViewArtifactsDirectoryName)
# Generate Swagger APIView Tokens
foreach ($entry in $autoRestConfigInfo.GetEnumerator()) {
$configInfo = $entry.Value
$readMeFile = $entry.Key
git checkout $SourceCommitId
if (Test-Path -Path $readmeFile) {
$resourceProvider = Get-ResourceProviderFromReadMePath -ReadMeFilePath $readMeFile
$tokenDirectory = [System.IO.Path]::Combine($swaggerAPIViewArtifactsDirectory, $resourceProvider)
New-Item -ItemType Directory -Path $tokenDirectory -Force | Out-Null
# Generate New APIView Token using default tag on source branch
Invoke-SwaggerAPIViewParser -Type "New" -ReadMeFilePath $readMeFile -ResourceProvider $resourceProvider -TokenDirectory $tokenDirectory `
-TempDirectory $TempDirectory -Tag $configInfo.DefaultTag | Out-Null
# Generate BaseLine APIView Token using source commit tag on target branch or defaukt tag if source commit tag does not exist
git checkout $TargetCommitId
if (Test-Path -Path $readMeFile) {
$targetTagInfo = Get-TagInformationFromReadMeFile -ReadMeFilePath $readMeFile
$baseLineTag = $targetTagInfo.DefaultTag
if ($targetTagInfo.Tags.Contains($configInfo.DefaultTag)) {
$baseLineTag = $configInfo.DefaultTag
}
Invoke-SwaggerAPIViewParser -Type "Baseline" -ReadMeFilePath $readMeFile -ResourceProvider $resourceProvider -TokenDirectory $tokenDirectory `
-TempDirectory $TempDirectory -Tag $baseLineTag | Out-Null
}
else {
LogWarning " Swagger ReadMe file '$readMeFile' not found on TargetBranch. Skipping APIView token generation."
}
}
else {
LogWarning " Swagger ReadMe file '$readMeFile' not found on SourceBranch. Skipping APIView token generation."
}
}
git checkout $currentBranch
$generatedSwaggerArtifacts = Get-ChildItem -Path $swaggerAPIViewArtifactsDirectory -Recurse
if ($generatedSwaggerArtifacts.Count -eq 0) {
LogWarning " No Swagger APIView Tokens generated..."
Write-Host "##vso[task.complete result=SucceededWithIssues;]DONE"
exit 0
}
LogGroupStart " See all generated Swagger APIView Artifacts..."
$generatedSwaggerArtifacts
LogGroupEnd
}
<#
.DESCRIPTION
Generate New and Baseline APIView tokens for the changed TypeSpec files in the PR.
Detects the TypeSpec files changed in the PR and generates APIView tokens for the TypeSpec files.
New APIView tokens are generated using the default tag on the base branch.
Baseline APIView tokens are generated using the same tag on the target branch.
Script asumes that the merge commit is checked out. Such that Source commit = HEAD^ and Target commit = HEAD.
.PARAMETER TempDirectory
Temporary directory for files being processed. Use $(Agent.TempDirectory) on DevOps
.PARAMETER ArtifactsStagingDirectory
The directory where the APIView tokens will be stored. Use $(Build.ArtifactStagingDirectory) on DevOps
.PARAMETER APIViewArtifactsDirectoryName
Name for the subdirectory where the APIView tokens will be stored.
#>
function New-TypeSpecAPIViewTokens {
param (
[Parameter(Mandatory = $true)]
[string]$TempDirectory,
[Parameter(Mandatory = $true)]
[string]$ArtifactsStagingDirectory,
[Parameter(Mandatory = $true)]
[string]$APIViewArtifactsDirectoryName
)
$SourceCommitId = $(git rev-parse HEAD^2)
$TargetCommitId = $(git rev-parse HEAD^1)
LogInfo " Getting changed TypeSpec files in PR, between $SourceCommitId and $TargetCommitId"
$changedFiles = Get-ChangedFiles
$changedTypeSpecFiles = Get-ChangedTypeSpecFiles -changedFiles $changedFiles
if ($changedTypeSpecFiles.Count -eq 0) {
LogWarning " There are no changes to TypeSpec files in the current PR..."
Write-Host "##vso[task.complete result=SucceededWithIssues;]DONE"
exit 0
}
LogGroupStart " Pullrequest has changes in these TypeSpec files..."
$changedTypeSpecFiles | ForEach-Object {
LogInfo " - $_"
}
LogGroupEnd
# Get impacted TypeSpec projects
$typeSpecProjects = [System.Collections.Generic.HashSet[string]]::new()
$changedTypeSpecFiles | ForEach-Object {
$tspProj = Get-ImpactedTypespecProjects -TypeSpecFile "$_"
if ($tspProj) {
$typeSpecProjects.Add($tspProj) | Out-Null
}
}
LogGroupStart " TypeSpec APIView Tokens will be generated for the following configuration files..."
$typeSpecProjects | ForEach-Object {
LogInfo " - $_"
}
LogGroupEnd
$currentBranch = git rev-parse --abbrev-ref HEAD
$typeSpecAPIViewArtifactsDirectory = [System.IO.Path]::Combine($ArtifactsStagingDirectory, $APIViewArtifactsDirectoryName)
New-Item -ItemType Directory -Path $typeSpecAPIViewArtifactsDirectory -Force | Out-Null
try {
npm --version --loglevel info
# Generate New TypeSpec APIView Tokens
git checkout $SourceCommitId
Write-Host "Installing required dependencies to generate New API review"
npm ci
LogGroupStart "npm ls -a"
npm ls -a
LogGroupEnd
foreach ($typeSpecProject in $typeSpecProjects) {
$tokenDirectory = Join-Path $typeSpecAPIViewArtifactsDirectory $(Split-Path $typeSpecProject -Leaf)
New-Item -ItemType Directory -Path $tokenDirectory -Force | Out-Null
Invoke-TypeSpecAPIViewParser -Type "New" -ProjectPath $typeSpecProject -ResourceProvider $(Split-Path $typeSpecProject -Leaf) -TokenDirectory $tokenDirectory
}
# Generate Baseline TypeSpec APIView Tokens
git checkout $TargetCommitId
Write-Host "Installing required dependencies to generate Baseline API review"
npm ci
LogGroupStart "npm ls -a"
npm ls -a
LogGroupEnd
foreach ($typeSpecProject in $typeSpecProjects) {
# Skip Baseline APIView Token for new projects
if (!(Test-Path -Path $typeSpecProject)) {
Write-Host "TypeSpec project $typeSpecProject is not found in pull request target branch. API review will not have a baseline revision."
}
else {
$tokenDirectory = Join-Path $typeSpecAPIViewArtifactsDirectory $(Split-Path $typeSpecProject -Leaf)
Invoke-TypeSpecAPIViewParser -Type "Baseline" -ProjectPath $typeSpecProject -ResourceProvider $(Split-Path $typeSpecProject -Leaf) -TokenDirectory $tokenDirectory | Out-Null
}
}
}
finally {
git checkout $currentBranch
LogGroupStart " See all generated TypeSpec APIView Artifacts..."
Get-ChildItem -Path $typeSpecAPIViewArtifactsDirectory -Recurse
LogGroupEnd
}
}
<#
.DESCRIPTION
Create APIView for the published packages. Send DevOps artifacts information to APIView to create APIView for the published packages.
.PARAMETER ArtifactsStagingDirectory
The DevOps artifacts staging directory. Use $(Build.ArtifactStagingDirectory) on DevOps
.PARAMETER APIViewArtifactsDirectoryName
Temporary Directory for processing the APIView artifacts
.PARAMETER APIViewArtifactsName
The name of the APIView artifact
.PARAMETER APIViewUri
The EndPoint for creating APIView https://apiviewstagingtest.com/PullRequest/DetectAPIChanges
.PARAMETER BuildId
TGhe BuildId of the Run
.PARAMETER RepoName
Repo name eg Azure/azure-rest-api-specs
.PARAMETER PullRequestNumber
The PR number
.PARAMETER Language
The language of the resource provider
.PARAMETER CommitSha
The commit sha of the current branch. Uusally the merge commit of the PR.
#>
function New-RestSpecsAPIViewReviews {
param (
[Parameter(Mandatory = $true)]
[string]$ArtifactsStagingDirectory,
[Parameter(Mandatory = $true)]
[string]$APIViewArtifactsDirectoryName,
[Parameter(Mandatory = $true)]
[string]$APIViewArtifactsName,
[Parameter(Mandatory = $true)]
[string]$APIViewUri,
[Parameter(Mandatory = $true)]
[string]$BuildId,
[Parameter(Mandatory = $true)]
[string]$RepoName,
[Parameter(Mandatory = $true)]
[string]$PullRequestNumber,
[Parameter(Mandatory = $true)]
[string]$Language,
[Parameter(Mandatory = $true)]
[string]$CommitSha
)
$apiViewArtifactsDirectory = [System.IO.Path]::Combine($ArtifactsStagingDirectory, $APIViewArtifactsDirectoryName)
$publishedPackages = Get-ChildItem -Path $apiViewArtifactsDirectory -Directory -ErrorAction SilentlyContinue
Write-Host "Published packages: $publishedPackages"
$createAPIViewFailed = $false
$publishedPackages | ForEach-Object {
$apiViewArtifacts = Get-ChildItem -Path $_.FullName -File -Filter "*.json" -ErrorAction SilentlyContinue
$query = [System.Web.HttpUtility]::ParseQueryString('')
$apiViewArtifacts | ForEach-Object {
if ($_.BaseName.EndsWith("New")) {
$query.Add("codeFile", $_.Name)
}
elseif ($_.BaseName.EndsWith("Baseline")) {
$query.Add("baselineCodeFile", $_.Name)
}
}
if (-not $query['codeFile']) {
LogWarning "'New' APIView token file not found for resource provider '$($_.BaseName)'. Skipping APIView creation."
return
}
if (-not $query['baselineCodeFile']) {
LogWarning "'Baseline' APIView token file not found for resource provider '$($_.BaseName)'. Created APIView without baseline."
}
$query.Add('artifactName', $APIViewArtifactsName)
$query.Add('buildId', $BuildId)
$query.Add('commitSha', $CommitSha)
$query.Add('repoName', $RepoName)
$query.Add('pullRequestNumber', $PullRequestNumber)
$query.Add('packageName', $_.BaseName)
$query.Add('language', $Language)
$query.Add('commentOnPR', $false)
$uri = [System.UriBuilder]$APIViewUri
$uri.Query = $query.ToString()
LogInfo "Create APIView for resource provider '$($_.BaseName)'"
LogInfo "APIView Uri: $($uri.Uri)"
try {
Invoke-WebRequest -Method 'GET' -Uri $uri.Uri -MaximumRetryCount 3
}
catch {
LogError "Failed to create APIView for resource provider '$($_.BaseName)'. Error: $($_.Exception.Response)"
$createAPIViewFailed = $true
}
}
if ($createAPIViewFailed) {
LogError "Failed to create APIView for some resource providers. Check the logs for more details."
Write-Host "##vso[task.complete result=SucceededWithIssues;]DONE"
exit 1
}
}
<#
.DESCRIPTION
Get all the tags from the swagger readme file with the default tag indicated.
.PARAMETER ReadMeFilePath
The file path to the readme file.
#>
function Get-TagInformationFromReadMeFile {
param (
[Parameter(Mandatory = $true)]
[string]$ReadMeFilePath
)
$tags = [System.Collections.Generic.HashSet[string]]::new()
$markDownContent = Get-Content -Path $ReadMeFilePath
$checkForDefaultTag = $false
$defaultTag = $null
foreach ($line in $markDownContent) {
$line = $line.Trim()
if ($line -match "###\s+Basic\s+Information") {
$checkForDefaultTag = $true
}
if ($checkForDefaulttag -and ($null -eq $defaultTag)) {
if ($line -match $defaultTagRegex) {
$defaultTag = $matches["tag"]
$checkForDefaultTag = $false
}
}
if ($line -match $tagRegex) {
$tag = $matches["tag"]
$tags.Add($tag) | Out-Null
}
}
[PSCustomObject]@{
Tags = $tags
DefaultTag = $defaultTag
}
}