eng/scripts/Language-Settings.ps1 (522 lines of code) (raw):

$Language = "dotnet" $LanguageShort = "net" $LanguageDisplayName = ".NET" $PackageRepository = "Nuget" $packagePattern = "*.nupkg" $MetadataUri = "https://raw.githubusercontent.com/Azure/azure-sdk/main/_data/releases/latest/dotnet-packages.csv" $GithubUri = "https://github.com/Azure/azure-sdk-for-net" $PackageRepositoryUri = "https://www.nuget.org/packages" . "$PSScriptRoot/docs/Docs-ToC.ps1" $DependencyCalculationPackages = @( "Azure.Core", "Azure.ResourceManager", "System.ClientModel" ) function processTestProject($projPath) { $cleanPath = $projPath -replace "^\$\(RepoRoot\)", "" # Split the path into segments $pathSegments = $cleanPath -split "[\\/]" # Find the index of the 'tests' directory $testsIndex = $pathSegments.IndexOf("tests") if ($testsIndex -gt 0) { # Reconstruct the path up to the directory just above 'tests' $parentDirectory = ($pathSegments[0..($testsIndex - 1)] -join [System.IO.Path]::DirectorySeparatorChar) return $parentDirectory } else { Write-Error "Unable to retrieve a package directory for this this test project: $projPath" # If 'tests' is not found, return the original path (optional behavior) $_ } } function Get-AllPackageInfoFromRepo($serviceDirectory) { $allPackageProps = @() # $addDevVersion is a global variable set by a parameter in # Save-Package-Properties.ps1 $shouldAddDevVersion = Get-Variable -Name 'addDevVersion' -ValueOnly -ErrorAction 'Ignore' $ServiceProj = Join-Path -Path $EngDir -ChildPath "service.proj" Write-Host "dotnet msbuild /nologo /t:GetPackageInfo ""$ServiceProj"" /p:ServiceDirectory=$serviceDirectory /p:AddDevVersion=$shouldAddDevVersion -tl:off" $msbuildOutput = dotnet msbuild ` /nologo ` /t:GetPackageInfo ` "$ServiceProj" ` /p:ServiceDirectory=$serviceDirectory ` /p:AddDevVersion=$shouldAddDevVersion ` -tl:off foreach ($projectOutput in $msbuildOutput) { if (!$projectOutput) { Write-Host "Get-AllPackageInfoFromRepo::projectOutput was null or empty, skipping" continue } $pkgPath, $serviceDirectory, $pkgName, $pkgVersion, $sdkType, $isNewSdk, $dllFolder = $projectOutput.Split("' '", [System.StringSplitOptions]::RemoveEmptyEntries).Trim("' ") if(!(Test-Path $pkgPath)) { Write-Host "Parsed package path `$pkgPath` does not exist so skipping the package line '$projectOutput'." continue } $pkgProp = [PackageProps]::new($pkgName, $pkgVersion, $pkgPath, $serviceDirectory) $pkgProp.SdkType = $sdkType $pkgProp.IsNewSdk = ($isNewSdk -eq 'true') $pkgProp.ArtifactName = $pkgName $pkgProp.IncludedForValidation = $false $pkgProp.DirectoryPath = ($pkgProp.DirectoryPath) if ($pkgProp.Name -in $DependencyCalculationPackages -and $env:DISCOVER_DEPENDENT_PACKAGES) { Write-Host "Calculating dependencies for $($pkgProp.Name)" $outputFilePath = Join-Path $RepoRoot "$($pkgProp.Name)_dependencylist.txt" $buildOutputPath = Join-Path $RepoRoot "$($pkgProp.Name)_dependencylistoutput.txt" if (!(Test-Path $outputFilePath)) { try { $command = "dotnet build /t:ProjectDependsOn ./eng/service.proj /p:TestDependsOnDependency=`"$($pkgProp.Name)`" /p:IncludeSrc=false " + "/p:IncludeStress=false /p:IncludeSamples=false /p:IncludePerf=false /p:RunApiCompat=false /p:InheritDocEnabled=false /p:BuildProjectReferences=false" + " /p:OutputProjectFilePath=`"$outputFilePath`" > $buildOutputPath 2>&1" Invoke-LoggedCommand $command | Out-Null } catch { Write-Host "Failed calculating dependencies for $($pkgProp.Name). Exit code $LASTEXITCODE." Write-Host "Dumping erroring build output." Write-Host (Get-Content -Raw $buildOutputPath) continue } } $pkgRelPath = $pkgProp.DirectoryPath.Replace($RepoRoot, "").TrimStart("\/") if (Test-Path $outputFilePath) { $dependentProjects = Get-Content $outputFilePath $testPackages = @{} foreach ($projectLine in $dependentProjects) { $testPackage = processTestProject($projectLine) if ($testPackage -and $testPackage -ne $pkgRelPath) { if ($testPackages[$testPackage]) { $testPackages[$testPackage] += 1 } else { $testPackages[$testPackage] = 1 } } } $pkgProp.AdditionalValidationPackages = $testPackages.Keys Write-Host "Cleaning up $outputFilePath." } } $ciProps = $pkgProp.GetCIYmlForArtifact() if ($ciProps) { # CheckAOTCompat is opt _in_, so we should default to false if not specified $shouldAot = GetValueSafelyFrom-Yaml $ciProps.ParsedYml @("extends", "parameters", "CheckAOTCompat") if ($null -ne $shouldAot) { $parsedBool = $null if ([bool]::TryParse($shouldAot, [ref]$parsedBool)) { $pkgProp.CIParameters["CheckAOTCompat"] = $parsedBool } # when AOTCompat is true, there is an additional parameter we need to retrieve $aotArtifacts = GetValueSafelyFrom-Yaml $ciProps.ParsedYml @("extends", "parameters", "AOTTestInputs") if ($aotArtifacts) { $aotArtifacts = $aotArtifacts | Where-Object { $_.ArtifactName -eq $pkgProp.ArtifactName } $pkgProp.CIParameters["AOTTestInputs"] = $aotArtifacts } } else { $pkgProp.CIParameters["CheckAOTCompat"] = $false $pkgProp.CIParameters["AOTTestInputs"] = @() } # BuildSnippets is opt _out_, so we should default to true if not specified $shouldSnippet = GetValueSafelyFrom-Yaml $ciProps.ParsedYml @("extends", "parameters", "BuildSnippets") if ($null -ne $shouldSnippet) { $parsedBool = $null if ([bool]::TryParse($shouldSnippet, [ref]$parsedBool)) { $pkgProp.CIParameters["BuildSnippets"] = $parsedBool } } else { $pkgProp.CIParameters["BuildSnippets"] = $true } } # if the package isn't associated with a CI.yml, we still want to set the defaults values for these parameters # so that when we are checking the package set for which need to "Build Snippets" or "Check AOT" we won't crash due to the property being null else { $pkgProp.CIParameters["CheckAOTCompat"] = $false $pkgProp.CIParameters["AOTTestInputs"] = @() $pkgProp.CIParameters["BuildSnippets"] = $true } $allPackageProps += $pkgProp } return $allPackageProps } # Returns the nuget publish status of a package id and version. function IsNugetPackageVersionPublished ($pkgId, $pkgVersion) { $nugetUri = "https://api.nuget.org/v3-flatcontainer/$($pkgId.ToLowerInvariant())/index.json" try { $nugetVersions = Invoke-RestMethod -MaximumRetryCount 3 -RetryIntervalSec 10 -uri $nugetUri -Method "GET" return $nugetVersions.versions.Contains($pkgVersion) } catch { $statusCode = $_.Exception.Response.StatusCode.value__ $statusDescription = $_.Exception.Response.ReasonPhrase # if this is 404ing, then this pkg has never been published before if ($statusCode -eq 404) { return $False } Write-Host "Nuget Invocation failed:" Write-Host "StatusCode:" $statusCode Write-Host "StatusDescription:" $statusDescription exit(1) } } # Parse out package publishing information given a nupkg ZIP format. function Get-dotnet-PackageInfoFromPackageFile ($pkg, $workingDirectory) { $workFolder = "$workingDirectory$($pkg.Basename)" $zipFileLocation = "$workFolder/$($pkg.Basename).zip" $releaseNotes = "" $readmeContent = "" New-Item -ItemType Directory -Force -Path $workFolder Copy-Item -Path $pkg -Destination $zipFileLocation Expand-Archive -Path $zipFileLocation -DestinationPath $workFolder [xml] $packageXML = Get-ChildItem -Path "$workFolder/*.nuspec" | Get-Content $pkgId = $packageXML.package.metadata.id $docsReadMeName = $pkgId -replace "^Azure." , "" $pkgVersion = $packageXML.package.metadata.version $changeLogLoc = @(Get-ChildItem -Path $workFolder -Recurse -Include "CHANGELOG.md")[0] if ($changeLogLoc) { $releaseNotes = Get-ChangeLogEntryAsString -ChangeLogLocation $changeLogLoc -VersionString $pkgVersion } $readmeContentLoc = @(Get-ChildItem -Path $workFolder -Recurse -Include "README.md")[0] if ($readmeContentLoc) { $readmeContent = Get-Content -Raw $readmeContentLoc } Remove-Item $workFolder -Force -Recurse -ErrorAction SilentlyContinue return New-Object PSObject -Property @{ PackageId = $pkgId PackageVersion = $pkgVersion ReleaseTag = "$($pkgId)_$($pkgVersion)" Deployable = $forceCreate -or !(IsNugetPackageVersionPublished -pkgId $pkgId -pkgVersion $pkgVersion) ReleaseNotes = $releaseNotes ReadmeContent = $readmeContent DocsReadMeName = $docsReadMeName } } # Return list of nupkg artifacts function Get-dotnet-Package-Artifacts ($Location) { $pkgs = @(Get-ChildItem $Location -Recurse | Where-Object -FilterScript {$_.Name.EndsWith(".nupkg") -and -not $_.Name.EndsWith(".symbols.nupkg")}) if (!$pkgs) { Write-Host "$($Location) does not have any package" return $null } elseif ($pkgs.Count -ne 1) { Write-Host "$($Location) should contain only one (1) published package" Write-Host "No of Packages $($pkgs.Count)" return $null } return $pkgs[0] } # Stage and Upload Docs to blob Storage function Publish-dotnet-GithubIODocs ($DocLocation, $PublicArtifactLocation) { $PublishedPkg = Get-dotnet-Package-Artifacts $DocLocation if (!$PublishedPkg) { Write-Host "Package is not available in artifact path $($DocLocation)" exit 1 } $PublishedDocs = Get-ChildItem "${DocLocation}" | Where-Object -FilterScript {$_.Name.EndsWith("docs.zip")} if ($PublishedDoc.Count -gt 1) { Write-Host "$($DocLocation) should contain only one (1) published package and docs" Write-Host "No of Docs $($PublishedDoc.Count)" exit 1 } $DocsStagingDir = "$WorkingDirectory/docstaging" $TempDir = "$WorkingDirectory/temp" New-Item -ItemType directory -Path $DocsStagingDir New-Item -ItemType directory -Path $TempDir Expand-Archive -LiteralPath $PublishedDocs[0].FullName -DestinationPath $DocsStagingDir $pkgProperties = Get-dotnet-PackageInfoFromPackageFile -pkg $PublishedPkg.FullName -workingDirectory $TempDir Write-Host "Start Upload for $($pkgProperties.ReleaseTag)" Write-Host "DocDir $($DocsStagingDir)" Write-Host "PkgName $($pkgProperties.PackageId)" Write-Host "DocVersion $($pkgProperties.PackageVersion)" Upload-Blobs -DocDir "$($DocsStagingDir)" -PkgName $pkgProperties.PackageId -DocVersion $pkgProperties.PackageVersion -ReleaseTag $pkgProperties.ReleaseTag } function Get-dotnet-GithubIoDocIndex() { # Update the main.js and docfx.json language content UpdateDocIndexFiles -appTitleLang $LanguageDisplayName # Fetch out all package metadata from csv file. $metadata = Get-CSVMetadata -MetadataUri $MetadataUri # Get the artifacts name from blob storage $artifacts = Get-BlobStorage-Artifacts ` -blobDirectoryRegex "^dotnet/(.*)/$" ` -blobArtifactsReplacement '$1' ` -storageAccountName 'azuresdkdocs' ` -storageContainerName '$web' ` -storagePrefix 'dotnet/' # Build up the artifact to service name mapping for GithubIo toc. $tocContent = Get-TocMapping -metadata $metadata -artifacts $artifacts # Generate yml/md toc files and build site. GenerateDocfxTocContent -tocContent $tocContent -lang $LanguageDisplayName -campaignId "UA-62780441-41" } # details on CSV schema can be found here # https://review.docs.microsoft.com/en-us/help/onboard/admin/reference/dotnet/documenting-nuget?branch=master#set-up-the-ci-job function Update-dotnet-CIConfig($pkgs, $ciRepo, $locationInDocRepo, $monikerId=$null) { $csvLoc = (Join-Path -Path $ciRepo -ChildPath $locationInDocRepo) if (-not (Test-Path $csvLoc)) { Write-Error "Unable to locate package csv at location $csvLoc, exiting." exit(1) } $allCSVRows = Get-Content $csvLoc $visibleInCI = @{} # first pull what's already available for ($i=0; $i -lt $allCSVRows.Length; $i++) { $pkgDef = $allCSVRows[$i] # get rid of the modifiers to get just the package id $id = $pkgDef.split(",")[1] -replace "\[.*?\]", "" $visibleInCI[$id] = $i } foreach ($releasingPkg in $pkgs) { $installModifiers = "tfm=netstandard2.0" if ($releasingPkg.IsPrerelease) { $installModifiers += ";isPrerelease=true" } $lineId = $releasingPkg.PackageId.Replace(".","").ToLower() if ($visibleInCI.ContainsKey($releasingPkg.PackageId)) { $packagesIndex = $visibleInCI[$releasingPkg.PackageId] $allCSVRows[$packagesIndex] = "$($lineId),[$installModifiers]$($releasingPkg.PackageId)" } else { $newItem = "$($lineId),[$installModifiers]$($releasingPkg.PackageId)" $allCSVRows += ($newItem) } } Set-Content -Path $csvLoc -Value $allCSVRows } # function is used to auto generate API View function Find-dotnet-Artifacts-For-Apireview($artifactDir, $packageName) { # Find all nupkg files in given artifact directory $PackageArtifactPath = Join-Path $artifactDir $packageName $pkg = Get-dotnet-Package-Artifacts $PackageArtifactPath if (!$pkg) { Write-Host "Package is not available in artifact path $($PackageArtifactPath)" return $null } $packages = @{ $pkg.Name = $pkg.FullName } return $packages } function SetPackageVersion ($PackageName, $Version, $ServiceDirectory, $ReleaseDate, $ReplaceLatestEntryTitle=$true) { if($null -eq $ReleaseDate) { $ReleaseDate = Get-Date -Format "yyyy-MM-dd" } & "$EngDir/scripts/Update-PkgVersion.ps1" -ServiceDirectory $ServiceDirectory -PackageName $PackageName ` -NewVersionString $Version -ReleaseDate $ReleaseDate -ReplaceLatestEntryTitle $ReplaceLatestEntryTitle } function GetExistingPackageVersions ($PackageName, $GroupId=$null) { try { $PackageName = $PackageName.ToLower() $existingVersion = Invoke-RestMethod -Method GET -Uri "https://api.nuget.org/v3-flatcontainer/${PackageName}/index.json" return $existingVersion.versions } catch { if ($_.Exception.Response.StatusCode -ne 404) { LogError "Failed to retrieve package versions for ${PackageName}. $($_.Exception.Message)" } return $null } } function Get-dotnet-DocsMsMetadataForPackage($PackageInfo) { $readmeName = $PackageInfo.Name.ToLower() # Readme names (which are used in the URL) should not include redundant terms # when viewed in URL form. For example: # https://docs.microsoft.com/en-us/dotnet/api/overview/azure/storage.blobs-readme # Note how the end of the URL doesn't look like: # ".../azure/azure.storage.blobs-readme" # This logic eliminates a preceeding "azure." in the readme filename. # "azure.storage.blobs" -> "storage.blobs" if ($readmeName.StartsWith('azure.')) { $readmeName = $readmeName.Substring(6) } New-Object PSObject -Property @{ DocsMsReadMeName = $readmeName LatestReadMeLocation = 'api/overview/azure/latest' PreviewReadMeLocation = 'api/overview/azure/preview' LegacyReadMeLocation = 'api/overview/azure/legacy' Suffix = '' } } # Details on CSV schema: # https://review.docs.microsoft.com/en-us/help/onboard/admin/reference/dotnet/documenting-nuget?branch=master#set-up-the-ci-job # # PowerShell's included Import-Csv cmdlet is not sufficient for parsing this # format because it does not easily handle rows whose number of columns is # greater than the number of columns in the first row. We must manually parse # this CSV file. function Get-DocsCiConfig($configPath) { Write-Host "Loading csv from $configPath" $output = @() foreach ($row in Get-Content $configPath) { # CSV format: # {package_moniker_base_string},{package_ID},{version_1},{version_2},...,{version_N} # # The {package_ID} field can contain optional properties denoted by square # brackets of the format: [key=value;key=value;...] # Split the rows by the comma $fields = $row.Split(',') if (!$fields -or $fields.Count -lt 2) { LogError "Please check the csv entry: $configPath." LogError "Do include the package name for each of the csv entry." } # If the {package_ID} field contains optional properties inside square # brackets, parse those properties into key value pairs. In the case of # duplicate keys, the last one wins. $rawProperties = '' $packageProperties = [ordered]@{} if ($fields[1] -match '\[(.*)\]') { $rawProperties = $Matches[1] foreach ($propertyExpression in $rawProperties.Split(';')) { $propertyParts = $propertyExpression.Split('=') $packageProperties[$propertyParts[0]] = $propertyParts[1] } } # Matches the "Package.Name" from the {package_ID} field. Possible # formats: # [key=value;key=value]Package.Name # Package.Name $packageName = '' if ($fields[1] -match '(\[.*\])?(.*)') { $packageName = $Matches[2] } else { Write-Error "Could not find package id in row: $row" } # Remaining entries in the row are versions, add them to the package # properties $outputVersions = @() if ($fields.Count -gt 2 -and $fields[2]) { $outputVersions = $fields[2..($fields.Count - 1)] } # Example row: # packagemoniker,[key1=value1;key2=value2]Package.Name,1.0.0,1.2.3-beta.1 $output += [PSCustomObject]@{ Id = $fields[0]; # packagemoniker Name = $packageName; # Package.Name Properties = $packageProperties; # @{key1='value1'; key2='value2'} Versions = $outputVersions # @('1.0.0', '1.2.3-beta.1') } } return $output } function EnsureCustomSource($package) { # $PackageSourceOverride is a global variable provided in # Update-DocsMsPackages.ps1. Its value can set a "customSource" property. # If it is empty then the property is not overridden. $customPackageSource = Get-Variable -Name 'PackageSourceOverride' -ValueOnly -ErrorAction 'Ignore' if (!$customPackageSource) { return $package } if (!(Get-PackageSource -Name CustomPackageSource -ErrorAction Ignore)) { Write-Host "Registering custom package source $customPackageSource" Register-PackageSource ` -Name CustomPackageSource ` -Location $customPackageSource ` -ProviderName NuGet ` -Force } Write-Host "Checking custom package source for $($package.Name)" try { $existingVersions = Find-Package ` -Name $package.Name ` -Source CustomPackageSource ` -AllVersions ` -AllowPrereleaseVersions if (!$? -or !$existingVersions) { Write-Host "Failed to find package $($package.Name) in custom source $customPackageSource" return $package } } catch { Write-Error $_ -ErrorAction Continue return $package } # Matches package version against output: # "Azure.Security.KeyVault.Secrets 4.3.0-alpha.20210915.3" $matchedVersion = $existingVersions.Where({$_.Version -eq $package.Versions}) if (!$matchedVersion) { return $package } $package.Properties['customSource'] = $customPackageSource return $package } function Get-dotnet-EmitterName() { return "@azure-tools/typespec-csharp" } function Get-dotnet-EmitterAdditionalOptions([string]$projectDirectory) { return "--option @azure-tools/typespec-csharp.emitter-output-dir=$projectDirectory/src" } function Update-dotnet-GeneratedSdks([string]$PackageDirectoriesFile) { Push-Location $RepoRoot try { Write-Host "`n`n======================================================================" Write-Host "Generating projects" -ForegroundColor Yellow Write-Host "======================================================================`n" $packageDirectories = Get-Content $PackageDirectoriesFile | ConvertFrom-Json # Build the project list override file $lines = @('<Project>', ' <ItemGroup>') foreach ($directory in $packageDirectories) { $projects = Get-ChildItem -Path "$RepoRoot/sdk/$directory" -Filter "*.csproj" -Recurse foreach ($project in $projects) { $lines += " <ProjectReference Include=`"$($project.FullName)`" />" } } $lines += ' </ItemGroup>', '</Project>' $artifactsPath = Join-Path $RepoRoot "artifacts" $projectListOverrideFile = Join-Path $artifactsPath "GeneratedSdks.proj" Write-Host "Creating project list override file $projectListOverrideFile`:" $lines | ForEach-Object { " $_" } | Out-Host New-Item $artifactsPath -ItemType Directory -Force | Out-Null $lines | Out-File $projectListOverrideFile -Encoding UTF8 Write-Host "`n" # Install autorest locally Invoke-LoggedCommand "npm ci --prefix $RepoRoot" Write-Host "Running npm ci over emitter-package.json in a temp folder to prime the npm cache" $tempFolder = New-TemporaryFile $tempFolder | Remove-Item -Force New-Item $tempFolder -ItemType Directory -Force | Out-Null Push-Location $tempFolder try { Copy-Item "$RepoRoot/eng/emitter-package.json" "package.json" if(Test-Path "$RepoRoot/eng/emitter-package-lock.json") { Copy-Item "$RepoRoot/eng/emitter-package-lock.json" "package-lock.json" Invoke-LoggedCommand "npm ci" } else { Invoke-LoggedCommand "npm install" } } finally { Pop-Location $tempFolder | Remove-Item -Force -Recurse } # Generate projects $showSummary = ($env:SYSTEM_DEBUG -eq 'true') -or ($VerbosePreference -ne 'SilentlyContinue') $summaryArgs = $showSummary ? "/v:n /ds" : "" Invoke-LoggedCommand "dotnet msbuild /restore /t:GenerateCode /p:ProjectListOverrideFile=$(Resolve-Path $projectListOverrideFile -Relative) $summaryArgs eng\service.proj" } finally { Pop-Location } } function Get-dotnet-ApiviewStatusCheckRequirement($packageInfo) { if ($packageInfo.IsNewSdk -and ($packageInfo.SdkType -eq "client" -or $packageInfo.SdkType -eq "mgmt")) { return $true } return $false }