SamplesV2/ContinuousIntegrationAndDelivery/PrePostDeploymentScript.Ver2.ps1 (702 lines of code) (raw):
<#
.SYNOPSIS
Stop/ start triggers during release process (CICD)
.DESCRIPTION
The script can be used to stop triggers before deployment and restart them afterward. It stops the trigger only if the trigger is in started state and there is a change in the trigger
.PARAMETER ArmTemplate
Arm template file path
example: C:\Adf\ArmTemlateOutput\ARMTemplateForFactory.json
.PARAMETER ResourceGroupName
Name of the resource group where the factory resource is in
.PARAMETER DataFactoryName
Name of the data factory being deployed
.PARAMETER PreDeployment
Default: $true
True: Runs script as pre-deployment so it will stop triggers prior to deployment
False: Runs script as post-deployment so it will delete the removed resources and start the triggers
.PARAMETER DeleteDeployment
Default: $false
Clean-up the deployment labels on the resource group
.PARAMETER ArmTemplateParameters
Default: $null
Arm template parameters file path
example: C:\Adf\ArmTemlateOutput\ARMTemplateParametersForFactory.json
If not provided, the script will try to find the file with name ARMTemplateParametersForFactory.json in ArmTemplate folder path
.PARAMETER ExplicitStopTriggerList
Default: @() (An empty array)
Expliticly stops the triggers form this list if the trigger is in started state even if the trigger payload not changed
example: "testTrigger", "storageEventsTrigger"
The script is not very comprehensive in detecting the trigger changes, so this parameter can be used to stop triggers explicitly if required
.PARAMETER MaxJsonDepth
Default: 100
The maximum depth the JSON input is allowed to have
#>
param
(
[parameter(Mandatory = $true)] [String] $ArmTemplate,
[parameter(Mandatory = $true)] [String] $ResourceGroupName,
[parameter(Mandatory = $true)] [String] $DataFactoryName,
[parameter(Mandatory = $false)] [Bool] $PreDeployment = $true,
[parameter(Mandatory = $false)] [Bool] $DeleteDeployment = $false,
[parameter(Mandatory = $false)] [String] $ArmTemplateParameters = $null,
[parameter(Mandatory = $false)] [String[]] $ExplicitStopTriggerList = @(),
[parameter(Mandatory = $false)] [int] $MaxJsonDepth = 100
)
function Get-PipelineDependency {
param(
[System.Object] $activity
)
$result = @()
if ($activity.Pipeline) {
$result += $activity.Pipeline.ReferenceName
}
elseif ($activity.Activities) {
$activity.Activities | ForEach-Object { $result += Get-PipelineDependency -activity $_ }
}
elseif ($activity.ifFalseActivities -or $activity.ifTrueActivities) {
$activity.ifFalseActivities | Where-Object { $_ -ne $null } | ForEach-Object { $result += Get-PipelineDependency -activity $_ }
$activity.ifTrueActivities | Where-Object { $_ -ne $null } | ForEach-Object { $result += Get-PipelineDependency -activity $_ }
}
elseif ($activity.defaultActivities) {
$activity.defaultActivities | ForEach-Object { $result += Get-PipelineDependency -activity $_ }
if ($activity.cases) {
$activity.cases | ForEach-Object { $_.activities } | ForEach-Object { $result += Get-PipelineDependency -activity $_ }
}
}
return $result
}
function Push-PipelinesToList {
param(
[Microsoft.Azure.Commands.DataFactoryV2.Models.PSPipeline]$pipeline,
[Hashtable] $pipelineNameResourceDict,
[Hashtable] $visited,
[System.Collections.Stack] $sortedList
)
if ($visited[$pipeline.Name] -eq $true) {
return;
}
$visited[$pipeline.Name] = $true;
$pipeline.Activities | ForEach-Object { Get-PipelineDependency -activity $_ -pipelineNameResourceDict $pipelineNameResourceDict } | ForEach-Object {
Push-PipelinesToList -pipeline $pipelineNameResourceDict[$_] -pipelineNameResourceDict $pipelineNameResourceDict -visited $visited -sortedList $sortedList
}
$sortedList.Push($pipeline)
}
function Get-SortedPipeline {
param(
[string] $DataFactoryName,
[string] $ResourceGroupName
)
$pipelines = Get-AzDataFactoryV2Pipeline -DataFactoryName $DataFactoryName -ResourceGroupName $ResourceGroupName
$ppDict = @{}
$visited = @{}
$stack = new-object System.Collections.Stack
$pipelines | ForEach-Object { $ppDict[$_.Name] = $_ }
$pipelines | ForEach-Object { Push-PipelinesToList -pipeline $_ -pipelineNameResourceDict $ppDict -visited $visited -sortedList $stack }
$sortedList = new-object Collections.Generic.List[Microsoft.Azure.Commands.DataFactoryV2.Models.PSPipeline]
while ($stack.Count -gt 0) {
$sortedList.Add($stack.Pop())
}
$sortedList
}
function Push-TriggersToList {
param(
[Microsoft.Azure.Commands.DataFactoryV2.Models.PSTrigger]$trigger,
[Hashtable] $triggerNameResourceDict,
[Hashtable] $visited,
[System.Collections.Stack] $sortedList
)
if ($visited[$trigger.Name] -eq $true) {
return;
}
$visited[$trigger.Name] = $true;
if ($trigger.Properties.DependsOn) {
$trigger.Properties.DependsOn | Where-Object { $_ -and $_.ReferenceTrigger } | ForEach-Object {
Push-TriggersToList -trigger $triggerNameResourceDict[$_.ReferenceTrigger.ReferenceName] -triggerNameResourceDict $triggerNameResourceDict -visited $visited -sortedList $sortedList
}
}
$sortedList.Push($trigger)
}
function Get-SortedTrigger {
param(
[string] $DataFactoryName,
[string] $ResourceGroupName
)
$triggers = Get-AzDataFactoryV2Trigger -ResourceGroupName $ResourceGroupName -DataFactoryName $DataFactoryName
$triggerDict = @{}
$visited = @{}
$stack = new-object System.Collections.Stack
$triggers | ForEach-Object { $triggerDict[$_.Name] = $_ }
$triggers | ForEach-Object { Push-TriggersToList -trigger $_ -triggerNameResourceDict $triggerDict -visited $visited -sortedList $stack }
$sortedList = new-object Collections.Generic.List[Microsoft.Azure.Commands.DataFactoryV2.Models.PSTrigger]
while ($stack.Count -gt 0) {
$sortedList.Add($stack.Pop())
}
$sortedList
}
function Get-SortedLinkedService {
param(
[string] $DataFactoryName,
[string] $ResourceGroupName
)
$linkedServices = Get-AzDataFactoryV2LinkedService -ResourceGroupName $ResourceGroupName -DataFactoryName $DataFactoryName
$LinkedServiceHasDependencies = @('HDInsightLinkedService', 'HDInsightOnDemandLinkedService', 'AzureBatchLinkedService')
$Akv = 'AzureKeyVaultLinkedService'
$HighOrderList = New-Object Collections.Generic.List[Microsoft.Azure.Commands.DataFactoryV2.Models.PSLinkedService]
$RegularList = New-Object Collections.Generic.List[Microsoft.Azure.Commands.DataFactoryV2.Models.PSLinkedService]
$AkvList = New-Object Collections.Generic.List[Microsoft.Azure.Commands.DataFactoryV2.Models.PSLinkedService]
$linkedServices | ForEach-Object {
if ($_.Properties.GetType().Name -in $LinkedServiceHasDependencies) {
$HighOrderList.Add($_)
}
elseif ($_.Properties.GetType().Name -eq $Akv) {
$AkvList.Add($_)
}
else {
$RegularList.Add($_)
}
}
$SortedList = New-Object Collections.Generic.List[Microsoft.Azure.Commands.DataFactoryV2.Models.PSLinkedService]($HighOrderList.Count + $RegularList.Count + $AkvList.Count)
$SortedList.AddRange($HighOrderList)
$SortedList.AddRange($RegularList)
$SortedList.AddRange($AkvList)
$SortedList
}
function Compare-TriggerPayload {
param(
[Microsoft.Azure.Commands.DataFactoryV2.Models.PSTrigger]$triggerDeployed,
[PSCustomObject]$triggerInTemplate,
[PSCustomObject]$templateParameters
)
try {
Write-Host "Comparing '$($triggerDeployed.Name)' trigger payload"
# Parse the trigger json from template to deserialize to trigger object
$triggerInTemplate.properties.typeProperties | Get-Member -MemberType NoteProperty | ForEach-Object {
$triggerInTemplate.properties | Add-Member -NotePropertyName $_.Name -NotePropertyValue $triggerInTemplate.properties.typeProperties.$($_.Name) -Force
}
$addPropDictionary = New-Object "System.Collections.Generic.Dictionary[System.String, System.Object]"
$addPropDictionary.Add('typeProperties', $triggerInTemplate.properties.typeProperties)
$triggerInTemplate.properties | Add-Member -NotePropertyName 'additionalProperties' -NotePropertyValue $addPropDictionary
$triggerInTemplate.properties.PSObject.Properties.Remove('typeProperties')
$triggerTemplateJson = ConvertTo-Json -InputObject $triggerInTemplate.properties -Depth $MaxJsonDepth -EscapeHandling Default
$updatedTemplateJson = Update-TriggerTemplate -templateJson $triggerTemplateJson -templateParameters $templateParameters
$serializerOptions = New-Object System.Text.Json.JsonSerializerOptions -Property @{ PropertyNameCaseInsensitive = $True }
$payloadPSObject = $updatedTemplateJson | ConvertFrom-Json -Depth $MaxJsonDepth
if ($triggerDeployed.Properties.RuntimeState -ne $payloadPSObject.runtimeState) {
Write-Host "Change detected in '$($triggerDeployed.Name)' trigger payload - runtimeState changed"
return $True;
}
if ($triggerDeployed.Properties.GetType().Name -eq [Microsoft.Azure.Management.DataFactory.Models.ScheduleTrigger].Name) {
# DayOfWeek needs to have enum value instead of enum strings
if ($payloadPSObject.recurrence.schedule.weekDays) {
[System.Array]$payloadPSObject.recurrence.schedule.weekDays = $payloadPSObject.recurrence.schedule.weekDays | ForEach-Object { ([System.DayOfWeek]::$_).value__ }
}
if ($payloadPSObject.recurrence.schedule.monthlyOccurrences) {
$payloadPSObject.recurrence.schedule.monthlyOccurrences | ForEach-Object { $_.day = ([System.DayOfWeek]::$($_.day)).value__ }
}
$updatedTemplateJson = ConvertTo-Json -InputObject $payloadPSObject -Depth $MaxJsonDepth
$triggerPayload = [System.Text.Json.JsonSerializer]::Deserialize($updatedTemplateJson,
[Microsoft.Azure.Management.DataFactory.Models.ScheduleTrigger],
$serializerOptions)
return Compare-ScheduleTrigger -triggerDeployed $triggerDeployed -triggerPayload $triggerPayload
}
elseif ($triggerDeployed.Properties.GetType().Name -eq [Microsoft.Azure.Management.DataFactory.Models.TumblingWindowTrigger].Name) {
$triggerPayload = [System.Text.Json.JsonSerializer]::Deserialize($updatedTemplateJson,
[Microsoft.Azure.Management.DataFactory.Models.TumblingWindowTrigger],
$serializerOptions)
return Compare-TumblingWindowTrigger -triggerDeployed $triggerDeployed -triggerPayload $triggerPayload
}
elseif ($triggerDeployed.Properties.GetType().Name -eq [Microsoft.Azure.Management.DataFactory.Models.BlobEventsTrigger].Name) {
$triggerPayload = [System.Text.Json.JsonSerializer]::Deserialize($updatedTemplateJson,
[Microsoft.Azure.Management.DataFactory.Models.BlobEventsTrigger],
$serializerOptions)
return Compare-BlobEventsTrigger -triggerDeployed $triggerDeployed -triggerPayload $triggerPayload
}
elseif ($triggerDeployed.Properties.GetType().Name -eq [Microsoft.Azure.Management.DataFactory.Models.CustomEventsTrigger].Name) {
$triggerPayload = [System.Text.Json.JsonSerializer]::Deserialize($updatedTemplateJson,
[Microsoft.Azure.Management.DataFactory.Models.CustomEventsTrigger],
$serializerOptions)
return Compare-CustomEventsTrigger -triggerDeployed $triggerDeployed -triggerPayload $triggerPayload
}
Write-Host "##[warning] Comparison terminated as trigger type '$($triggerDeployed.Properties.GetType().Name)' not match"
return $True
}
catch {
Write-Host "##[warning] Unable to compare '$($triggerDeployed.Name)' trigger payload, this is not a failure. You can post the issue to https://github.com/Azure/Azure-DataFactory/issues to check if this is user error or limitation."
Write-Host "##[warning] $_ from Line: $($_.InvocationInfo.ScriptLineNumber)"
return $True;
}
}
function Compare-ScheduleTrigger {
param(
[Microsoft.Azure.Commands.DataFactoryV2.Models.PSTrigger]$triggerDeployed,
[Microsoft.Azure.Management.DataFactory.Models.ScheduleTrigger]$triggerPayload
)
# Compare if any common trigger properties changed
$deployedTriggerProps = $triggerDeployed.Properties
$descriptionChanges = Compare-Object -ReferenceObject $deployedTriggerProps -DifferenceObject $triggerPayload -Property Description
$annotationChanges = Compare-Object -ReferenceObject $deployedtriggerProps.Annotations -DifferenceObject $triggerPayload.Annotations
# Compare if the recurrence changed
$recurrencechanges = Compare-Object -ReferenceObject $deployedTriggerProps.Recurrence -DifferenceObject $triggerPayload.Recurrence `
-Property Frequency, Interval, StartTime, EndTime, TimeZone
# Compare if the schedule changed
$scheduleChanged = $True;
if ($null -ne $deployedTriggerProps.Recurrence.Schedule -and $null -ne $triggerPayload.Recurrence.Schedule) {
$changes = Compare-Object -ReferenceObject $deployedTriggerProps.Recurrence.Schedule -DifferenceObject $triggerPayload.Recurrence.Schedule `
-Property Minutes, Hours, WeekDays, MonthDays
# Compare monthly occurrences
if ($null -eq $changes) {
$scheduleChanged = $True;
if ($null -ne $deployedTriggerProps.Recurrence.Schedule.MonthlyOccurrences -and $null -ne $triggerPayload.Recurrence.Schedule.MonthlyOccurrences) {
$changes =Compare-Object -ReferenceObject $deployedTriggerProps.Recurrence.Schedule.MonthlyOccurrences -DifferenceObject $triggerPayload.Recurrence.Schedule.MonthlyOccurrences `
-Property Day, Occurrence
} elseif ($null -eq $deployedTriggerProps.Recurrence.Schedule.MonthlyOccurrences -and $null -eq $triggerPayload.Recurrence.Schedule.MonthlyOccurrences) {
$scheduleChanged = $False;
}
}
$scheduleChanged = $null -ne $changes
} elseif ($null -eq $deployedTriggerProps.Recurrence.Schedule -and $null -eq $triggerPayload.Recurrence.Schedule) {
$scheduleChanged = $False;
}
# Compare to check if there is any change in referenced pipeline
$pipelineRefChanged = Compare-TriggerPipelineReference -tprDeployed $deployedTriggerProps.Pipelines -tprPayload $triggerPayload.Pipelines
# Temporary workaround for Get-AzDataFactoryV2Trigger returning incorrect datetime notation
# https://github.com/Azure/azure-powershell/issues/20474 issue, thanks to gvdmaaden for this workaround
Update-TriggerTimeFormat -deployedAdditionalProps $deployedTriggerProps.AdditionalProperties
# Compare additional properties (unmatched properties stay here)
$additionalPropsChanged = Compare-TriggerAdditionalProperty -deployedAdditionalProps $deployedTriggerProps.AdditionalProperties `
-payloadAdditionalProps $triggerPayload.AdditionalProperties
if (($null -ne $descriptionChanges) -or ($null -ne $annotationChanges) -or ($null -ne $recurrencechanges) -or `
$scheduleChanged -or $pipelineRefChanged -or $additionalPropsChanged) {
Write-Host "Change detected in '$($triggerDeployed.Name)' trigger payload - descriptionChanges=$($descriptionChanges.Length), annotationChanges=$($annotationChanges.Length), recurrencechanges=$($recurrencechanges.Length), scheduleChanged=$scheduleChanged, pipelineRefChanged=$pipelineRefChanged, additionalPropsChanged=$additionalPropsChanged"
return $True
}
Write-Host "No change detected in '$($triggerDeployed.Name)' trigger payload"
return $False;
}
function Compare-TumblingWindowTrigger {
param(
[Microsoft.Azure.Commands.DataFactoryV2.Models.PSTrigger]$triggerDeployed,
[Microsoft.Azure.Management.DataFactory.Models.TumblingWindowTrigger]$triggerPayload
)
# Compare if any of common tumbling window trigger properties changed
$propertyChanges = Compare-Object -ReferenceObject $triggerDeployed.Properties -DifferenceObject $triggerPayload `
-Property Frequency, Interval, StartTime, EndTime, Delay, MaxConcurrency, Description
$annotationChanges = Compare-Object -ReferenceObject $triggerDeployed.Properties.Annotations -DifferenceObject $triggerPayload.Annotations
$retryPolicyChanges = Compare-Object -ReferenceObject $triggerDeployed.Properties.RetryPolicy -DifferenceObject $triggerPayload.RetryPolicy `
-Property Count, IntervalInSeconds
# Compare to check if there is any change in referenced pipeline
$tprDeployed = New-Object System.Collections.Generic.List[Microsoft.Azure.Management.DataFactory.Models.TriggerPipelineReference]
$tprDeployed.Add($triggerDeployed.Properties.Pipeline)
$tprPayload = New-Object System.Collections.Generic.List[Microsoft.Azure.Management.DataFactory.Models.TriggerPipelineReference]
$tprPayload.Add($triggerPayload.Pipeline)
$pipelineRefChanged = Compare-TriggerPipelineReference -tprDeployed $tprDeployed -tprPayload $tprPayload
# Compare additional properties (unmatched properties stay here ex: DependsOn)
$additionalPropsChanged = Compare-TriggerAdditionalProperty -deployedAdditionalProps $triggerDeployed.Properties.AdditionalProperties `
-payloadAdditionalProps $triggerPayload.AdditionalProperties
if (($null -ne $propertyChanges) -or ($null -ne $annotationChanges) -or ($null -ne $retryPolicyChanges) -or `
$pipelineRefChanged -or $additionalPropsChanged) {
Write-Host "Change detected in '$($triggerDeployed.Name)' trigger payload - propertyChanges=$($propertyChanges.Length), annotationChanges=$($annotationChanges.Length), retryPolicyChanges=$($retryPolicyChanges.Length), pipelineRefChanged=$pipelineRefChanged, additionalPropsChanged=$additionalPropsChanged"
return $True
}
Write-Host "No change detected in '$($triggerDeployed.Name)' trigger payload"
return $False;
}
function Compare-BlobEventsTrigger {
param(
[Microsoft.Azure.Commands.DataFactoryV2.Models.PSTrigger]$triggerDeployed,
[Microsoft.Azure.Management.DataFactory.Models.BlobEventsTrigger]$triggerPayload
)
$propertyChanges = Compare-Object -ReferenceObject $triggerDeployed.Properties -DifferenceObject $triggerPayload `
-Property BlobPathBeginsWith, BlobPathEndsWith, IgnoreEmptyBlobs, Events, Scope, Description
$annotationChanges = Compare-Object -ReferenceObject $triggerDeployed.Properties.Annotations -DifferenceObject $triggerPayload.Annotations
# Compare to check if there is any change in referenced pipeline
$pipelineRefChanged = Compare-TriggerPipelineReference -tprDeployed $triggerDeployed.Properties.Pipelines -tprPayload $triggerPayload.Pipelines
# Compare additional properties (unmatched properties stay here - ex: advancedFilters)
$additionalPropsChanged = Compare-TriggerAdditionalProperty -deployedAdditionalProps $triggerDeployed.Properties.AdditionalProperties `
-payloadAdditionalProps $triggerPayload.AdditionalProperties
if (($null -ne $propertyChanges) -or ($null -ne $annotationChanges) -or $pipelineRefChanged -or $additionalPropsChanged) {
Write-Host "Change detected in '$($triggerDeployed.Name)' trigger payload - propertyChanges=$($propertyChanges.Length), annotationChanges=$($annotationChanges.Length), pipelineRefChanged=$pipelineRefChanged, additionalPropsChanged=$additionalPropsChanged"
return $True
}
Write-Host "No change detected in '$($triggerDeployed.Name)' trigger payload"
return $False;
}
function Compare-CustomEventsTrigger {
param(
[Microsoft.Azure.Commands.DataFactoryV2.Models.PSTrigger]$triggerDeployed,
[Microsoft.Azure.Management.DataFactory.Models.CustomEventsTrigger]$triggerPayload
)
# Compare common and event properties
$propertyChanges = Compare-Object -ReferenceObject $triggerDeployed.Properties -DifferenceObject $triggerPayload `
-Property SubjectBeginsWith, SubjectEndsWith, Scope, Description
$eventChanges = Compare-Object -ReferenceObject $triggerDeployed.Properties.Events -DifferenceObject $triggerPayload.Events
$annotationChanges = Compare-Object -ReferenceObject $triggerDeployed.Properties.Annotations -DifferenceObject $triggerPayload.Annotations
# Compare to check if there is any change in referenced pipeline
$pipelineRefChanged = Compare-TriggerPipelineReference -tprDeployed $triggerDeployed.Properties.Pipelines -tprPayload $triggerPayload.Pipelines
# Compare additional properties (unmatched properties stay here - ex: advancedFilters)
$additionalPropsChanged = Compare-TriggerAdditionalProperty -deployedAdditionalProps $triggerDeployed.Properties.AdditionalProperties `
-payloadAdditionalProps $triggerPayload.AdditionalProperties
if (($null -ne $propertyChanges) -or ($null -ne $eventChanges) -or ($null -ne $annotationChanges) -or `
$pipelineRefChanged -or $additionalPropsChanged) {
Write-Host "Change detected in '$($triggerDeployed.Name)' trigger payload - propertyChanges=$($propertyChanges.Length), eventChanges=$($eventChanges.Length), annotationChanges=$($annotationChanges.Length), pipelineRefChanged=$pipelineRefChanged, additionalPropsChanged=$additionalPropsChanged"
return $True
}
Write-Host "No change detected in '$($triggerDeployed.Name)' trigger payload"
return $False;
}
function Compare-TriggerPipelineReference {
param(
[System.Collections.Generic.IList[Microsoft.Azure.Management.DataFactory.Models.TriggerPipelineReference]]$tprDeployed,
[System.Collections.Generic.IList[Microsoft.Azure.Management.DataFactory.Models.TriggerPipelineReference]]$tprPayload
)
# Compare to check if there is any change in referenced pipeline
$pipelineRefchanged = $True;
if ($null -ne $tprDeployed.PipelineReference -and $null -ne $tprPayload.PipelineReference) {
$changes = Compare-Object -ReferenceObject $tprDeployed.PipelineReference -DifferenceObject $tprPayload.PipelineReference `
-Property Name, ReferenceName
$pipelineRefchanged = $changes.Length -gt 0
}
elseif ($null -eq $tprDeployed.PipelineReference -and $null -eq $tprPayload.PipelineReference) {
$pipelineRefchanged = $False;
}
# If no change in pipeline reference, compare to check if there is any change in pipeline parameters
$paramsChanged = $True
if (!$pipelineRefchanged -and $tprPayload.Count -gt 0) {
$paramsChanged = $False
for ($counter = 0; $counter -lt $tprPayload.count; $counter++) {
$pipelineReferenceName = $tprPayload[$counter].PipelineReference.ReferenceName
$payloadPipelineRef = $tprPayload | Where-Object { $_.PipelineReference.ReferenceName -eq $pipelineReferenceName }
$deployedPipelineRef = $tprDeployed | Where-Object { $_.PipelineReference.ReferenceName -eq $pipelineReferenceName }
if ($deployedPipelineRef.Parameters.Keys.Count -eq $payloadPipelineRef.Parameters.Keys.Count) {
foreach ($key in $deployedPipelineRef.Parameters.Keys) {
$deployedValue = $null
$payloadValue = $null
if (!$payloadPipelineRef.Parameters.TryGetValue($key, [ref]$payloadValue) -or
!$deployedPipelineRef.Parameters.TryGetValue($key, [ref]$deployedValue) ) {
$paramsChanged = $True
break
}
else {
if ($deployedValue.GetType().Name -in @("JObject", "JArray")) {
if (($deployedValue.ToString() | ConvertFrom-Json).type -eq "SecureString") {
$paramsChanged = $True
Write-Host "##[warning] SecureString parameter is always treated as a change"
break
} else {
$paramValueChanges = Compare-Object -ReferenceObject ($deployedValue.ToString() | ConvertFrom-Json) -DifferenceObject ($payloadValue.ToString() | ConvertFrom-Json)
}
} elseif ($deployedValue.GetType().Name -eq "Boolean") {
$paramValueChanges = Compare-Object -ReferenceObject ($deployedValue.ToString().ToLower() | ConvertFrom-Json) -DifferenceObject ($payloadValue.ToString().ToLower() | ConvertFrom-Json)
} else {
$paramValueChanges = Compare-Object -ReferenceObject $deployedValue -DifferenceObject $payloadValue
}
if ($paramValueChanges.Length -gt 0) {
$paramsChanged = $True
break
}
}
}
}
else {
$paramsChanged = $True
break;
}
}
}
return $pipelineRefchanged -or $paramsChanged
}
function Compare-TriggerAdditionalProperty {
param(
[System.Collections.Generic.Dictionary[String, System.Object]]$deployedAdditionalProps,
[System.Collections.Generic.Dictionary[String, System.Object]]$payloadAdditionalProps
)
$additionalPropchanged = $True;
if ($null -ne $deployedAdditionalProps -and $null -ne $payloadAdditionalProps) {
$changes = Compare-Object -ReferenceObject $deployedAdditionalProps -DifferenceObject $payloadAdditionalProps `
-Property Keys
$additionalPropchanged = $null -ne $changes
if (-not $additionalPropchanged) {
foreach ($key in $deployedAdditionalProps.Keys) {
$deployedValue = $null; $payloadValue = $null;
if (!$payloadAdditionalProps.TryGetValue($key, [ref]$payloadValue) -or
!$deployedAdditionalProps.TryGetValue($key, [ref]$deployedValue)) {
$additionalPropchanged = $True
break
}
else {
$payloadJObect = [Newtonsoft.Json.Linq.JObject]::Parse($payloadValue)
$additionalPropValueChanges = Compare-Object -ReferenceObject $deployedValue -DifferenceObject $payloadJObect
if ($null -ne $additionalPropValueChanges) {
$additionalPropchanged = $True
break
}
}
}
}
}
elseif ($null -eq $deployedAdditionalProps -and $null -eq $payloadAdditionalProps) {
$additionalPropchanged = $False;
}
return $additionalPropchanged
}
function Update-TriggerTimeFormat {
param(
[System.Collections.Generic.Dictionary[String, System.Object]]$deployedAdditionalProps
)
foreach ($key in $deployedAdditionalProps.Keys) {
$deployedValue = $null;
if ($deployedAdditionalProps.TryGetValue($key, [ref]$deployedValue) -and $deployedValue["recurrence"]["timeZone"] -ne "UTC")
{
if ($null -ne $deployedValue["recurrence"]["startTime"]) {
$startTimeString = $deployedValue["recurrence"]["startTime"].ToString("yyyy-MM-ddTHH:mm:ss");
$startDateTime = Get-Date -Date $startTimeString;
$deployedValue["recurrence"]["startTime"] = New-Object DateTime 2000, 1, 1, 1, 0, 0, ([DateTimeKind]::Unspecified)
$deployedValue["recurrence"]["startTime"] = New-Object DateTime $startDateTime.Year, $startDateTime.Month, $startDateTime.Day, $startDateTime.Hour, $startDateTime.Minute, $startDateTime.Second, ([DateTimeKind]::Unspecified)
}
if ($null -ne $deployedValue["recurrence"]["endTime"]) {
$endTimeString = $deployedValue["recurrence"]["endTime"].ToString("yyyy-MM-ddTHH:mm:ss");
$endDateTime = Get-Date -Date $endTimeString;
$deployedValue["recurrence"]["endTime"] = New-Object DateTime 2000, 1, 1, 1, 0, 0, ([DateTimeKind]::Unspecified)
$deployedValue["recurrence"]["endTime"] = New-Object DateTime $endDateTime.Year, $endDateTime.Month, $endDateTime.Day, $endDateTime.Hour, $endDateTime.Minute, $endDateTime.Second, ([DateTimeKind]::Unspecified)
}
}
}
}
function Update-TriggerTemplate {
param(
[string]$templateJson,
[PSCustomObject]$templateParameters
)
$parameterMatches = [System.Text.RegularExpressions.Regex]::Matches($templateJson, '\[parameters\([^)]*\)\]')
foreach ($parameterMatch in $parameterMatches) {
[string]$parameterName = $parameterMatch.Value.Substring(13, $parameterMatch.Value.Length - 16)
if ($null -ne $templateParameters.$($parameterName) && ($PreDeployment || $parameterName.EndsWith('runtimeState'))) {
$parameterType = $null -ne $templateParameters.$($parameterName).value ? $templateParameters.$($parameterName).value.GetType().Name : $null
if ($parameterType -eq 'Object[]' -or $parameterType -eq 'PSCustomObject') {
$parameterValue = ConvertTo-Json $templateParameters.$($parameterName).value -Depth $MaxJsonDepth -EscapeHandling Default
$templateJson = $templateJson -replace [System.Text.RegularExpressions.Regex]::Escape("`"$($parameterMatch.Value)`""), $parameterValue
} elseif ($parameterType -eq 'Int64') {
$templateJson = $templateJson -replace [System.Text.RegularExpressions.Regex]::Escape("`"$($parameterMatch.Value)`""), $templateParameters.$($parameterName).value
} elseif ($parameterType -eq 'Boolean') {
$templateJson = $templateJson -replace [System.Text.RegularExpressions.Regex]::Escape("`"$($parameterMatch.Value)`""), $templateParameters.$($parameterName).value.ToString().ToLower()
} else {
$templateJson = $templateJson -replace [System.Text.RegularExpressions.Regex]::Escape($parameterMatch.Value), $templateParameters.$($parameterName).value
}
}
}
return $templateJson
}
try {
# Show warning message that PowerShell Core or PowerShell version > 7.0 is required
$PSCompatible = $True
if ($PSVersionTable.PSEdition -ne 'Core' -and ([System.Version]$PSVersionTable.PSVersion -lt [System.Version]"7.0.0")) {
$PSCompatible = $False
Write-Host "##[warning] The script is not compatible with your current PowerShell version $($PSVersionTable.PSVersion). Use either PowerShell Core or at least PS version 7.0, otherwise the script may fail to compare the trigger payload and always stop/start the trigger(s)"
}
$templateJson = Get-Content $ArmTemplate | ConvertFrom-Json
$resources = $templateJson.resources
if (-not $ArmTemplateParameters) {
$ArmTemplateParameters = Join-Path -Path (Split-Path $ArmTemplate -Parent) -ChildPath 'ArmTemplateParametersForFactory.json'
Write-Host "##[warning] Arm-template parameter file path not specified, the script will look for the file in arm-template file path."
}
$templateParameters = $null
if (Test-Path -Path $ArmTemplateParameters) {
$templateParametersJson = Get-Content $ArmTemplateParameters | ConvertFrom-Json
$templateParameters = $templateParametersJson.parameters
} else {
Write-Host "##[warning] The script couldn't find the arm-tempalte parameter file in the arm-template file path, the trigger comparison won't work for parameterized properties. Please pass the arm-template parameter file path to ArmTemplateParameters script argument."
}
#Triggers
Write-Host "Getting triggers"
$triggersInTemplate = $resources | Where-Object { $_.type -eq "Microsoft.DataFactory/factories/triggers" }
$triggerNamesInTemplate = $triggersInTemplate | ForEach-Object { $_.name.Substring(37, $_.name.Length - 40) }
$triggersDeployed = Get-SortedTrigger -DataFactoryName $DataFactoryName -ResourceGroupName $ResourceGroupName
if ($PreDeployment -eq $true) {
#Stop trigger only if there is change in payload
$triggersToStop = $triggersDeployed | Where-Object { $_.Name -in $triggerNamesInTemplate -and $_.RuntimeState -ne 'Stopped' } `
| Where-Object {
$triggerName = $_.Name;
$triggerInTemplate = $triggersInTemplate | Where-Object { $_.name.Substring(37, $_.name.Length - 40) -eq $triggerName };
Compare-TriggerPayload -triggerDeployed $_ -triggerInTemplate $triggerInTemplate -templateParameters $templateParameters
} `
| ForEach-Object {
New-Object PSObject -Property @{
Name = $_.Name
TriggerType = $_.Properties.GetType().Name
}
}
Write-Host "Stopping $($triggersToStop.Count) triggers `n"
$triggersToStop | ForEach-Object {
if ($_.TriggerType -eq 'BlobEventsTrigger') {
Write-Host "Unsubscribing $($_.Name) from events"
$status = Remove-AzDataFactoryV2TriggerSubscription -ResourceGroupName $ResourceGroupName -DataFactoryName $DataFactoryName -Name $_.Name
while ($status.Status -ne 'Disabled') {
Start-Sleep -s 15
$status = Get-AzDataFactoryV2TriggerSubscriptionStatus -ResourceGroupName $ResourceGroupName -DataFactoryName $DataFactoryName -Name $_.Name
}
}
Write-Host "Stopping trigger $($_.Name)"
Stop-AzDataFactoryV2Trigger -ResourceGroupName $ResourceGroupName -DataFactoryName $DataFactoryName -Name $_.Name -Force
}
$explicitTriggersToStop = $triggersDeployed | Where-Object { $_.Name -in $triggerNamesInTemplate -and $_.RuntimeState -ne 'Stopped' } `
| Where-Object { $_.Name -in $ExplicitStopTriggerList } `
| ForEach-Object {
New-Object PSObject -Property @{
Name = $_.Name
TriggerType = $_.Properties.GetType().Name
}
}
if ($explicitTriggersToStop -and $explicitTriggersToStop.Count -gt 0) {
Write-Host "Stopping $($explicitTriggersToStop.Count) triggers from explicit stop-trigger list `n"
$explicitTriggersToStop | ForEach-Object {
if ($_.TriggerType -eq 'BlobEventsTrigger') {
Write-Host "Unsubscribing $($_.Name) from events"
$status = Remove-AzDataFactoryV2TriggerSubscription -ResourceGroupName $ResourceGroupName -DataFactoryName $DataFactoryName -Name $_.Name
while ($status.Status -ne 'Disabled') {
Start-Sleep -s 15
$status = Get-AzDataFactoryV2TriggerSubscriptionStatus -ResourceGroupName $ResourceGroupName -DataFactoryName $DataFactoryName -Name $_.Name
}
}
Write-Host "Stopping trigger $($_.Name)"
Stop-AzDataFactoryV2Trigger -ResourceGroupName $ResourceGroupName -DataFactoryName $DataFactoryName -Name $_.Name -Force
}
} elseif ($ExplicitStopTriggerList -and $ExplicitStopTriggerList.Count -gt 0) {
Write-Host "No matching trigger (in started state) to stop from explicit stop-trigger list"
}
}
else {
#Deleted resources
#pipelines
Write-Host "Getting pipelines"
$pipelinesADF = Get-SortedPipeline -DataFactoryName $DataFactoryName -ResourceGroupName $ResourceGroupName
$pipelinesTemplate = $resources | Where-Object { $_.type -eq "Microsoft.DataFactory/factories/pipelines" }
$pipelinesNames = $pipelinesTemplate | ForEach-Object { $_.name.Substring(37, $_.name.Length - 40) }
$deletedpipelines = $pipelinesADF | Where-Object { $pipelinesNames -notcontains $_.Name }
#dataflows
$dataflowsADF = Get-AzDataFactoryV2DataFlow -DataFactoryName $DataFactoryName -ResourceGroupName $ResourceGroupName
$dataflowsTemplate = $resources | Where-Object { $_.type -eq "Microsoft.DataFactory/factories/dataflows" }
$dataflowsNames = $dataflowsTemplate | ForEach-Object { $_.name.Substring(37, $_.name.Length - 40) }
$deleteddataflow = $dataflowsADF | Where-Object { $dataflowsNames -notcontains $_.Name }
#datasets
Write-Host "Getting datasets"
$datasetsADF = Get-AzDataFactoryV2Dataset -DataFactoryName $DataFactoryName -ResourceGroupName $ResourceGroupName
$datasetsTemplate = $resources | Where-Object { $_.type -eq "Microsoft.DataFactory/factories/datasets" }
$datasetsNames = $datasetsTemplate | ForEach-Object { $_.name.Substring(37, $_.name.Length - 40) }
$deleteddataset = $datasetsADF | Where-Object { $datasetsNames -notcontains $_.Name }
#linkedservices
Write-Host "Getting linked services"
$linkedservicesADF = Get-SortedLinkedService -DataFactoryName $DataFactoryName -ResourceGroupName $ResourceGroupName
$linkedservicesTemplate = $resources | Where-Object { $_.type -eq "Microsoft.DataFactory/factories/linkedservices" }
$linkedservicesNames = $linkedservicesTemplate | ForEach-Object { $_.name.Substring(37, $_.name.Length - 40) }
$deletedlinkedservices = $linkedservicesADF | Where-Object { $linkedservicesNames -notcontains $_.Name }
#Integrationruntimes
Write-Host "Getting integration runtimes"
$integrationruntimesADF = Get-AzDataFactoryV2IntegrationRuntime -DataFactoryName $DataFactoryName -ResourceGroupName $ResourceGroupName
$integrationruntimesTemplate = $resources | Where-Object { $_.type -eq "Microsoft.DataFactory/factories/integrationruntimes" }
$integrationruntimesNames = $integrationruntimesTemplate | ForEach-Object { $_.name.Substring(37, $_.name.Length - 40) }
$deletedintegrationruntimes = $integrationruntimesADF | Where-Object { $integrationruntimesNames -notcontains $_.Name }
#Delete resources
Write-Host "Deleting triggers"
$triggersToDelete = $triggersDeployed | Where-Object { $triggerNamesInTemplate -notcontains $_.Name } | ForEach-Object {
New-Object PSObject -Property @{
Name = $_.Name
TriggerType = $_.Properties.GetType().Name
}
}
$triggersToDelete | ForEach-Object {
Write-Host "Deleting trigger $($_.Name)"
$trig = Get-AzDataFactoryV2Trigger -name $_.Name -ResourceGroupName $ResourceGroupName -DataFactoryName $DataFactoryName
if ($trig.RuntimeState -eq 'Started') {
if ($_.TriggerType -eq 'BlobEventsTrigger') {
Write-Host "Unsubscribing trigger $($_.Name) from events"
$status = Remove-AzDataFactoryV2TriggerSubscription -ResourceGroupName $ResourceGroupName -DataFactoryName $DataFactoryName -Name $_.Name
while ($status.Status -ne 'Disabled') {
Start-Sleep -s 15
$status = Get-AzDataFactoryV2TriggerSubscriptionStatus -ResourceGroupName $ResourceGroupName -DataFactoryName $DataFactoryName -Name $_.Name
}
}
Stop-AzDataFactoryV2Trigger -ResourceGroupName $ResourceGroupName -DataFactoryName $DataFactoryName -Name $_.Name -Force
}
Remove-AzDataFactoryV2Trigger -Name $_.Name -ResourceGroupName $ResourceGroupName -DataFactoryName $DataFactoryName -Force
}
Write-Host "Deleting pipelines"
$deletedpipelines | ForEach-Object {
Write-Host "Deleting pipeline $($_.Name)"
Remove-AzDataFactoryV2Pipeline -Name $_.Name -ResourceGroupName $ResourceGroupName -DataFactoryName $DataFactoryName -Force
}
Write-Host "Deleting dataflows"
$deleteddataflow | ForEach-Object {
Write-Host "Deleting dataflow $($_.Name)"
Remove-AzDataFactoryV2DataFlow -Name $_.Name -ResourceGroupName $ResourceGroupName -DataFactoryName $DataFactoryName -Force
}
Write-Host "Deleting datasets"
$deleteddataset | ForEach-Object {
Write-Host "Deleting dataset $($_.Name)"
Remove-AzDataFactoryV2Dataset -Name $_.Name -ResourceGroupName $ResourceGroupName -DataFactoryName $DataFactoryName -Force
}
Write-Host "Deleting linked services"
$deletedlinkedservices | ForEach-Object {
Write-Host "Deleting Linked Service $($_.Name)"
Remove-AzDataFactoryV2LinkedService -Name $_.Name -ResourceGroupName $ResourceGroupName -DataFactoryName $DataFactoryName -Force
}
Write-Host "Deleting integration runtimes"
$deletedintegrationruntimes | ForEach-Object {
Write-Host "Deleting integration runtime $($_.Name)"
Remove-AzDataFactoryV2IntegrationRuntime -Name $_.Name -ResourceGroupName $ResourceGroupName -DataFactoryName $DataFactoryName -Force
}
if ($DeleteDeployment -eq $true) {
Write-Host "Deleting ARM deployment ... under resource group: $ResourceGroupName"
$deployments = Get-AzResourceGroupDeployment -ResourceGroupName $ResourceGroupName
$deploymentsToConsider = $deployments | Where-Object { $_.DeploymentName -like "ArmTemplate_master*" -or $_.DeploymentName -like "ArmTemplateForFactory*" } | Sort-Object -Property Timestamp -Descending
$deploymentName = $deploymentsToConsider[0].DeploymentName
Write-Host "Deployment to be deleted: $deploymentName"
$deploymentOperations = Get-AzResourceGroupDeploymentOperation -DeploymentName $deploymentName -ResourceGroupName $ResourceGroupName
$deploymentsToDelete = $deploymentOperations | Where-Object { $_.properties.targetResource.id -like "*Microsoft.Resources/deployments*" }
$deploymentsToDelete | ForEach-Object {
Write-Host "Deleting inner deployment: $($_.properties.targetResource.id)"
Remove-AzResourceGroupDeployment -Id $_.properties.targetResource.id
}
Write-Host "Deleting deployment: $deploymentName"
Remove-AzResourceGroupDeployment -ResourceGroupName $ResourceGroupName -Name $deploymentName
}
#Start active triggers - after cleanup efforts
$triggersRunning = $triggersDeployed | Where-Object { $_.RuntimeState -eq 'Started' } | ForEach-Object { $_.Name }
$updatedTriggersInTemplate = $triggersInTemplate
if ($PSCompatible) {
try {
$updatedTriggersInTemplate = $triggersInTemplate | ForEach-Object {
$triggerJson = ConvertTo-Json -InputObject $_ -Depth $MaxJsonDepth -EscapeHandling Default
Update-TriggerTemplate -templateJson $triggerJson -templateParameters $templateParameters
} | ConvertFrom-Json -Depth $MaxJsonDepth
} catch {
Write-Host "##[warning] Unable to update the parameterized properties in trigger template. The script may fail to start triggers if using parameterized runtimeState."
$updatedTriggersInTemplate = $triggersInTemplate
}
}
$triggersToStart = $updatedTriggersInTemplate | Where-Object { $_.properties.runtimeState -eq 'Started' -and $_.name.Substring(37, $_.name.Length - 40) -notin $triggersRunning } `
| Where-Object { $_.properties.pipelines.Count -gt 0 -or $_.properties.pipeline.pipelineReference -ne $null } | ForEach-Object {
New-Object PSObject -Property @{
Name = $_.name.Substring(37, $_.name.Length - 40)
TriggerType = $_.Properties.type
}
}
if ($triggersToStart.Count -gt 0) {
Write-Host "Starting $($triggersToStart.Count) triggers"
}
$triggersToStart | ForEach-Object {
if ($_.TriggerType -eq 'BlobEventsTrigger') {
Write-Host "Subscribing $($_.Name) to events"
$status = Add-AzDataFactoryV2TriggerSubscription -ResourceGroupName $ResourceGroupName -DataFactoryName $DataFactoryName -Name $_.Name
while ($status.Status -ne 'Enabled') {
Start-Sleep -s 15
$status = Get-AzDataFactoryV2TriggerSubscriptionStatus -ResourceGroupName $ResourceGroupName -DataFactoryName $DataFactoryName -Name $_.Name
}
}
Write-Host "Starting trigger $($_.Name)"
Start-AzDataFactoryV2Trigger -ResourceGroupName $ResourceGroupName -DataFactoryName $DataFactoryName -Name $_.Name -Force
}
}
} catch {
Write-Host "##[error] $_ from Line: $($_.InvocationInfo.ScriptLineNumber)"
throw
}