From be3bae81fc0c54afe2a32845ef28a352d21c6add Mon Sep 17 00:00:00 2001 From: MaksimZhukov Date: Mon, 15 Jun 2020 12:29:35 +0300 Subject: [PATCH 1/5] Improve builds runner --- azure-devops/run-ci-builds.ps1 | 99 ++++++++++++++++++++++++++-------- 1 file changed, 78 insertions(+), 21 deletions(-) diff --git a/azure-devops/run-ci-builds.ps1 b/azure-devops/run-ci-builds.ps1 index bf11295..0b1c9de 100644 --- a/azure-devops/run-ci-builds.ps1 +++ b/azure-devops/run-ci-builds.ps1 @@ -3,31 +3,80 @@ param ( [Parameter(Mandatory)] [string] $AzureDevOpsProjectName, [Parameter(Mandatory)] [string] $AzureDevOpsAccessToken, [Parameter(Mandatory)] [string] $SourceBranch, - [Parameter(Mandatory)] [string] $ToolVersions, [Parameter(Mandatory)] [UInt32] $DefinitionId, - [string] $SourceVersion + [Parameter(Mandatory)] [string] $SourceVersion, + [Parameter(Mandatory)] [string] $ManifestLink, + [Parameter(Mandatory)] [bool] $WaitForBuilds, + [string] $ToolVersions, + [UInt32] $RetryIntervalSec = 30, + [UInt32] $RetryCount = 20 ) Import-Module (Join-Path $PSScriptRoot "azure-devops-api.ps1") Import-Module (Join-Path $PSScriptRoot "build-info.ps1") +function Get-ToolVersions { + param ( + [Parameter(Mandatory)] [string] $ManifestLink, + [Parameter(Mandatory)] [UInt32] $RetryIntervalSec, + [Parameter(Mandatory)] [UInt32] $Retries, + [string] $ToolVersions + ) + + [string[]] $versionsList = @() + if ($ToolVersions){ + $versionsList = $ToolVersions.Split(',') + } else { + Write-Host "Get the list of releases from $ManifestLink" + $releases = Invoke-RestMethod $ManifestLink -MaximumRetryCount $Retries -RetryIntervalSec $RetryIntervalSec + $releases | ForEach-Object { $versionsList += $_.version } + } + + return $versionsList +} + function Queue-Builds { param ( [Parameter(Mandatory)] [AzureDevOpsApi] $AzureDevOpsApi, - [Parameter(Mandatory)] [string] $ToolVersions, + [Parameter(Mandatory)] [string[]] $ToolVersions, [Parameter(Mandatory)] [string] $SourceBranch, [Parameter(Mandatory)] [string] $SourceVersion, - [Parameter(Mandatory)] [string] $DefinitionId + [Parameter(Mandatory)] [UInt32] $DefinitionId, + [Parameter(Mandatory)] [UInt32] $RetryIntervalSec, + [Parameter(Mandatory)] [UInt32] $Retries ) [BuildInfo[]]$queuedBuilds = @() - $ToolVersions.Split(',') | ForEach-Object { + $ToolVersions | ForEach-Object { $version = $_.Trim() - Write-Host "Queue build for $version..." - $queuedBuild = $AzureDevOpsApi.QueueBuild($version, $SourceBranch, $SourceVersion, $DefinitionId) - $buildInfo = Get-BuildInfo -AzureDevOpsApi $AzureDevOpsApi -Build $queuedBuild - Write-Host "Queued build: $($buildInfo.Link)" + + while ($Retries -gt 0) + { + try + { + Write-Host "Queue build for $version..." + $queuedBuild = $AzureDevOpsApi.QueueBuild($version, $SourceBranch, $SourceVersion, $DefinitionId) + $buildInfo = Get-BuildInfo -AzureDevOpsApi $AzureDevOpsApi -Build $queuedBuild + Write-Host "Queued build: $($buildInfo.Link)" + break + } + catch + { + Write-Host "There is an error during build starting:`n $_" + $Retries-- + + if ($Retries -eq 0) + { + Write-Host "Build can't be queued, please try later" + exit 1 + } + + Write-Host "Waiting 30 seconds before retrying. Retries left: $Retries" + Start-Sleep -Seconds $RetryIntervalSec + } + } + $queuedBuilds += $buildInfo } @@ -36,10 +85,9 @@ function Queue-Builds { function Wait-Builds { param ( - [Parameter(Mandatory)] [BuildInfo[]] $Builds + [Parameter(Mandatory)] [BuildInfo[]] $Builds, + [Parameter(Mandatory)] [UInt32] $RetryIntervalSec ) - - $timeoutBetweenRefreshSec = 30 do { # If build is still running - refresh its status @@ -55,7 +103,7 @@ function Wait-Builds { $runningBuildsCount = ($builds | Where-Object { !$_.IsFinished() }).Length - Start-Sleep -Seconds $timeoutBetweenRefreshSec + Start-Sleep -Seconds $RetryIntervalSec } while($runningBuildsCount -gt 0) } @@ -64,7 +112,7 @@ function Make-BuildsOutput { [Parameter(Mandatory)] [BuildInfo[]] $Builds ) - Write-Host "Builds info:" + Write-Host "`nBuilds info:" $builds | Format-Table -AutoSize -Property Name,Id,Status,Result,Link | Out-String -Width 10000 # Return exit code based on status of builds @@ -74,7 +122,7 @@ function Make-BuildsOutput { $failedBuilds | ForEach-Object -Process { Write-Host "##vso[task.logissue type=error;]Name: $($_.Name); Link: $($_.Link)" } Write-Host "##vso[task.complete result=Failed]" } else { - Write-host "##[section] All builds have been passed successfully" + Write-host "##[section]All builds have been passed successfully" } } @@ -82,13 +130,22 @@ $azureDevOpsApi = Get-AzureDevOpsApi -TeamFoundationCollectionUri $TeamFoundatio -ProjectName $AzureDevOpsProjectName ` -AccessToken $AzureDevOpsAccessToken +$toolVersionsList = Get-ToolVersions -ManifestLink $ManifestLink ` + -RetryIntervalSec $RetryIntervalSec ` + -Retries $RetryCount ` + -ToolVersions $ToolVersions + $queuedBuilds = Queue-Builds -AzureDevOpsApi $azureDevOpsApi ` - -ToolVersions $ToolVersions ` + -ToolVersions $toolVersionsList ` -SourceBranch $SourceBranch ` -SourceVersion $SourceVersion ` - -DefinitionId $DefinitionId - -Write-Host "Waiting results of builds ..." -Wait-Builds -Builds $queuedBuilds + -DefinitionId $DefinitionId ` + -RetryIntervalSec $RetryIntervalSec ` + -Retries $RetryCount -Make-BuildsOutput -Builds $queuedBuilds +if ($WaitForBuilds) { + Write-Host "`nWaiting results of builds ..." + Wait-Builds -Builds $queuedBuilds -RetryIntervalSec $RetryIntervalSec + + Make-BuildsOutput -Builds $queuedBuilds +} From 93258f50880b5d5af5f514b28bbece53f0b51bbd Mon Sep 17 00:00:00 2001 From: MaksimZhukov Date: Mon, 15 Jun 2020 15:29:28 +0300 Subject: [PATCH 2/5] Remove ForEach --- azure-devops/run-ci-builds.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-devops/run-ci-builds.ps1 b/azure-devops/run-ci-builds.ps1 index 0b1c9de..ab67672 100644 --- a/azure-devops/run-ci-builds.ps1 +++ b/azure-devops/run-ci-builds.ps1 @@ -24,12 +24,12 @@ function Get-ToolVersions { ) [string[]] $versionsList = @() - if ($ToolVersions){ + if ($ToolVersions) { $versionsList = $ToolVersions.Split(',') } else { Write-Host "Get the list of releases from $ManifestLink" $releases = Invoke-RestMethod $ManifestLink -MaximumRetryCount $Retries -RetryIntervalSec $RetryIntervalSec - $releases | ForEach-Object { $versionsList += $_.version } + $versionsList = $releases.version } return $versionsList From b0414ba6277e324cd3a5f480c2e8dd000736c5fe Mon Sep 17 00:00:00 2001 From: MaksimZhukov Date: Mon, 15 Jun 2020 23:02:04 +0300 Subject: [PATCH 3/5] Move retry logic to the AzureDevOpsApi class --- azure-devops/azure-devops-api.ps1 | 23 +++++++++++++--- azure-devops/run-ci-builds.ps1 | 44 +++++++------------------------ 2 files changed, 29 insertions(+), 38 deletions(-) diff --git a/azure-devops/azure-devops-api.ps1 b/azure-devops/azure-devops-api.ps1 index dd2feba..7d6ee21 100644 --- a/azure-devops/azure-devops-api.ps1 +++ b/azure-devops/azure-devops-api.ps1 @@ -3,14 +3,25 @@ class AzureDevOpsApi [string] $BaseUrl [string] $RepoOwner [object] $AuthHeader + [UInt32] $RetryCount = 0 + [UInt32] $RetryIntervalSec = 30 AzureDevOpsApi( [string] $TeamFoundationCollectionUri, [string] $ProjectName, - [string] $AccessToken + [string] $AccessToken, + [UInt32] $RetryCount, + [UInt32] $RetryIntervalSec ) { $this.BaseUrl = $this.BuildBaseUrl($TeamFoundationCollectionUri, $ProjectName) $this.AuthHeader = $this.BuildAuth($AccessToken) + + if ($RetryCount -gt 0) { + $this.RetryCount = $RetryCount + } + if ($RetryIntervalSec -gt 0) { + $this.RetryIntervalSec = $RetryIntervalSec + } } [object] hidden BuildAuth([string]$AccessToken) { @@ -73,17 +84,21 @@ class AzureDevOpsApi $params.Body = $Body } + $params.RetryIntervalSec = $this.RetryIntervalSec + $params.MaximumRetryCount = $this.RetryCount + return Invoke-RestMethod @params } - } function Get-AzureDevOpsApi { param ( [string] $TeamFoundationCollectionUri, [string] $ProjectName, - [string] $AccessToken + [string] $AccessToken, + [UInt32] $RetryCount, + [UInt32] $RetryIntervalSec ) - return [AzureDevOpsApi]::New($TeamFoundationCollectionUri, $ProjectName, $AccessToken) + return [AzureDevOpsApi]::New($TeamFoundationCollectionUri, $ProjectName, $AccessToken, $RetryCount, $RetryIntervalSec) } \ No newline at end of file diff --git a/azure-devops/run-ci-builds.ps1 b/azure-devops/run-ci-builds.ps1 index ab67672..9480ef4 100644 --- a/azure-devops/run-ci-builds.ps1 +++ b/azure-devops/run-ci-builds.ps1 @@ -32,6 +32,7 @@ function Get-ToolVersions { $versionsList = $releases.version } + Write-Host "Versions to build: $versionsList" return $versionsList } @@ -41,42 +42,17 @@ function Queue-Builds { [Parameter(Mandatory)] [string[]] $ToolVersions, [Parameter(Mandatory)] [string] $SourceBranch, [Parameter(Mandatory)] [string] $SourceVersion, - [Parameter(Mandatory)] [UInt32] $DefinitionId, - [Parameter(Mandatory)] [UInt32] $RetryIntervalSec, - [Parameter(Mandatory)] [UInt32] $Retries + [Parameter(Mandatory)] [UInt32] $DefinitionId ) [BuildInfo[]]$queuedBuilds = @() $ToolVersions | ForEach-Object { $version = $_.Trim() - - while ($Retries -gt 0) - { - try - { - Write-Host "Queue build for $version..." - $queuedBuild = $AzureDevOpsApi.QueueBuild($version, $SourceBranch, $SourceVersion, $DefinitionId) - $buildInfo = Get-BuildInfo -AzureDevOpsApi $AzureDevOpsApi -Build $queuedBuild - Write-Host "Queued build: $($buildInfo.Link)" - break - } - catch - { - Write-Host "There is an error during build starting:`n $_" - $Retries-- - - if ($Retries -eq 0) - { - Write-Host "Build can't be queued, please try later" - exit 1 - } - - Write-Host "Waiting 30 seconds before retrying. Retries left: $Retries" - Start-Sleep -Seconds $RetryIntervalSec - } - } - + Write-Host "Queue build for $version..." + $queuedBuild = $AzureDevOpsApi.QueueBuild($version, $SourceBranch, $SourceVersion, $DefinitionId) + $buildInfo = Get-BuildInfo -AzureDevOpsApi $AzureDevOpsApi -Build $queuedBuild + Write-Host "Queued build: $($buildInfo.Link)" $queuedBuilds += $buildInfo } @@ -128,7 +104,9 @@ function Make-BuildsOutput { $azureDevOpsApi = Get-AzureDevOpsApi -TeamFoundationCollectionUri $TeamFoundationCollectionUri ` -ProjectName $AzureDevOpsProjectName ` - -AccessToken $AzureDevOpsAccessToken + -AccessToken $AzureDevOpsAccessToken ` + -RetryCount $RetryCount ` + -RetryIntervalSec $RetryIntervalSec $toolVersionsList = Get-ToolVersions -ManifestLink $ManifestLink ` -RetryIntervalSec $RetryIntervalSec ` @@ -139,9 +117,7 @@ $queuedBuilds = Queue-Builds -AzureDevOpsApi $azureDevOpsApi ` -ToolVersions $toolVersionsList ` -SourceBranch $SourceBranch ` -SourceVersion $SourceVersion ` - -DefinitionId $DefinitionId ` - -RetryIntervalSec $RetryIntervalSec ` - -Retries $RetryCount + -DefinitionId $DefinitionId if ($WaitForBuilds) { Write-Host "`nWaiting results of builds ..." From ca5ee91832bf1897a6a8710f516d3ba032448ce6 Mon Sep 17 00:00:00 2001 From: MaksimZhukov Date: Tue, 16 Jun 2020 14:03:26 +0300 Subject: [PATCH 4/5] Change RetryCount --- azure-devops/azure-devops-api.ps1 | 17 ++++++----------- azure-devops/run-ci-builds.ps1 | 2 +- 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/azure-devops/azure-devops-api.ps1 b/azure-devops/azure-devops-api.ps1 index 7d6ee21..b4c35e0 100644 --- a/azure-devops/azure-devops-api.ps1 +++ b/azure-devops/azure-devops-api.ps1 @@ -3,8 +3,8 @@ class AzureDevOpsApi [string] $BaseUrl [string] $RepoOwner [object] $AuthHeader - [UInt32] $RetryCount = 0 - [UInt32] $RetryIntervalSec = 30 + [UInt32] $RetryCount + [UInt32] $RetryIntervalSec AzureDevOpsApi( [string] $TeamFoundationCollectionUri, @@ -15,13 +15,8 @@ class AzureDevOpsApi ) { $this.BaseUrl = $this.BuildBaseUrl($TeamFoundationCollectionUri, $ProjectName) $this.AuthHeader = $this.BuildAuth($AccessToken) - - if ($RetryCount -gt 0) { - $this.RetryCount = $RetryCount - } - if ($RetryIntervalSec -gt 0) { - $this.RetryIntervalSec = $RetryIntervalSec - } + $this.RetryCount = $RetryCount + $this.RetryIntervalSec = $RetryIntervalSec } [object] hidden BuildAuth([string]$AccessToken) { @@ -96,8 +91,8 @@ function Get-AzureDevOpsApi { [string] $TeamFoundationCollectionUri, [string] $ProjectName, [string] $AccessToken, - [UInt32] $RetryCount, - [UInt32] $RetryIntervalSec + [UInt32] $RetryCount = 3, + [UInt32] $RetryIntervalSec = 30 ) return [AzureDevOpsApi]::New($TeamFoundationCollectionUri, $ProjectName, $AccessToken, $RetryCount, $RetryIntervalSec) diff --git a/azure-devops/run-ci-builds.ps1 b/azure-devops/run-ci-builds.ps1 index 9480ef4..349db47 100644 --- a/azure-devops/run-ci-builds.ps1 +++ b/azure-devops/run-ci-builds.ps1 @@ -9,7 +9,7 @@ param ( [Parameter(Mandatory)] [bool] $WaitForBuilds, [string] $ToolVersions, [UInt32] $RetryIntervalSec = 30, - [UInt32] $RetryCount = 20 + [UInt32] $RetryCount = 3 ) Import-Module (Join-Path $PSScriptRoot "azure-devops-api.ps1") From 6543512f08b5e73db75746ba59beddcdd6aa709e Mon Sep 17 00:00:00 2001 From: MaksimZhukov Date: Tue, 16 Jun 2020 14:37:52 +0300 Subject: [PATCH 5/5] Increase RetryIntervalSec --- azure-devops/azure-devops-api.ps1 | 2 +- azure-devops/run-ci-builds.ps1 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-devops/azure-devops-api.ps1 b/azure-devops/azure-devops-api.ps1 index b4c35e0..bb5ed1e 100644 --- a/azure-devops/azure-devops-api.ps1 +++ b/azure-devops/azure-devops-api.ps1 @@ -92,7 +92,7 @@ function Get-AzureDevOpsApi { [string] $ProjectName, [string] $AccessToken, [UInt32] $RetryCount = 3, - [UInt32] $RetryIntervalSec = 30 + [UInt32] $RetryIntervalSec = 60 ) return [AzureDevOpsApi]::New($TeamFoundationCollectionUri, $ProjectName, $AccessToken, $RetryCount, $RetryIntervalSec) diff --git a/azure-devops/run-ci-builds.ps1 b/azure-devops/run-ci-builds.ps1 index 349db47..cd93837 100644 --- a/azure-devops/run-ci-builds.ps1 +++ b/azure-devops/run-ci-builds.ps1 @@ -8,7 +8,7 @@ param ( [Parameter(Mandatory)] [string] $ManifestLink, [Parameter(Mandatory)] [bool] $WaitForBuilds, [string] $ToolVersions, - [UInt32] $RetryIntervalSec = 30, + [UInt32] $RetryIntervalSec = 60, [UInt32] $RetryCount = 3 )