From be3bae81fc0c54afe2a32845ef28a352d21c6add Mon Sep 17 00:00:00 2001
From: MaksimZhukov <v-mazhuk@microsoft.com>
Date: Mon, 15 Jun 2020 12:29:35 +0300
Subject: [PATCH] Improve builds runner

---
 azure-devops/run-ci-builds.ps1 | 99 ++++++++++++++++++++++++++--------
 1 file changed, 78 insertions(+), 21 deletions(-)

diff --git a/azure-devops/run-ci-builds.ps1 b/azure-devops/run-ci-builds.ps1
index bf11295..0b1c9de 100644
--- a/azure-devops/run-ci-builds.ps1
+++ b/azure-devops/run-ci-builds.ps1
@@ -3,31 +3,80 @@ param (
     [Parameter(Mandatory)] [string] $AzureDevOpsProjectName,
     [Parameter(Mandatory)] [string] $AzureDevOpsAccessToken,
     [Parameter(Mandatory)] [string] $SourceBranch,
-    [Parameter(Mandatory)] [string] $ToolVersions,
     [Parameter(Mandatory)] [UInt32] $DefinitionId,
-    [string] $SourceVersion
+    [Parameter(Mandatory)] [string] $SourceVersion,
+    [Parameter(Mandatory)] [string] $ManifestLink,
+    [Parameter(Mandatory)] [bool] $WaitForBuilds,
+    [string] $ToolVersions,
+    [UInt32] $RetryIntervalSec = 30,
+    [UInt32] $RetryCount = 20
 )
 
 Import-Module (Join-Path $PSScriptRoot "azure-devops-api.ps1")
 Import-Module (Join-Path $PSScriptRoot "build-info.ps1")
 
+function Get-ToolVersions {
+    param (
+        [Parameter(Mandatory)] [string] $ManifestLink,
+        [Parameter(Mandatory)] [UInt32] $RetryIntervalSec,
+        [Parameter(Mandatory)] [UInt32] $Retries,
+        [string] $ToolVersions
+    )
+    
+    [string[]] $versionsList = @()
+    if ($ToolVersions){
+        $versionsList = $ToolVersions.Split(',')
+    } else {
+        Write-Host "Get the list of releases from $ManifestLink"
+        $releases = Invoke-RestMethod $ManifestLink -MaximumRetryCount $Retries -RetryIntervalSec $RetryIntervalSec
+        $releases | ForEach-Object { $versionsList += $_.version }
+    }
+
+    return $versionsList
+}
+
 function Queue-Builds {
     param (
         [Parameter(Mandatory)] [AzureDevOpsApi] $AzureDevOpsApi,
-        [Parameter(Mandatory)] [string] $ToolVersions,
+        [Parameter(Mandatory)] [string[]] $ToolVersions,
         [Parameter(Mandatory)] [string] $SourceBranch,
         [Parameter(Mandatory)] [string] $SourceVersion,
-        [Parameter(Mandatory)] [string] $DefinitionId
+        [Parameter(Mandatory)] [UInt32] $DefinitionId,
+        [Parameter(Mandatory)] [UInt32] $RetryIntervalSec,
+        [Parameter(Mandatory)] [UInt32] $Retries
     )
 
     [BuildInfo[]]$queuedBuilds = @()
 
-    $ToolVersions.Split(',') | ForEach-Object { 
+    $ToolVersions | ForEach-Object { 
         $version = $_.Trim()
-        Write-Host "Queue build for $version..."
-        $queuedBuild = $AzureDevOpsApi.QueueBuild($version, $SourceBranch, $SourceVersion, $DefinitionId)
-        $buildInfo = Get-BuildInfo -AzureDevOpsApi $AzureDevOpsApi -Build $queuedBuild
-        Write-Host "Queued build: $($buildInfo.Link)"
+        
+        while ($Retries -gt 0)
+        {
+            try
+            {
+                Write-Host "Queue build for $version..."
+                $queuedBuild = $AzureDevOpsApi.QueueBuild($version, $SourceBranch, $SourceVersion, $DefinitionId)
+                $buildInfo = Get-BuildInfo -AzureDevOpsApi $AzureDevOpsApi -Build $queuedBuild
+                Write-Host "Queued build: $($buildInfo.Link)"
+                break
+            }
+            catch
+            {
+                Write-Host "There is an error during build starting:`n $_"
+                $Retries--
+            
+                if ($Retries -eq 0)
+                {
+                    Write-Host "Build can't be queued, please try later"
+                    exit 1
+                }
+            
+                Write-Host "Waiting 30 seconds before retrying. Retries left: $Retries"
+                Start-Sleep -Seconds $RetryIntervalSec
+            }
+        }
+        
         $queuedBuilds += $buildInfo
     }
 
@@ -36,10 +85,9 @@ function Queue-Builds {
 
 function Wait-Builds {
     param (
-        [Parameter(Mandatory)] [BuildInfo[]] $Builds
+        [Parameter(Mandatory)] [BuildInfo[]] $Builds,
+        [Parameter(Mandatory)] [UInt32] $RetryIntervalSec
     )
-
-    $timeoutBetweenRefreshSec = 30
     
     do {
         # If build is still running - refresh its status
@@ -55,7 +103,7 @@ function Wait-Builds {
     
         $runningBuildsCount = ($builds | Where-Object { !$_.IsFinished() }).Length
 
-        Start-Sleep -Seconds $timeoutBetweenRefreshSec
+        Start-Sleep -Seconds $RetryIntervalSec
     } while($runningBuildsCount -gt 0)
 }
 
@@ -64,7 +112,7 @@ function Make-BuildsOutput {
         [Parameter(Mandatory)] [BuildInfo[]] $Builds
     )
 
-    Write-Host "Builds info:"
+    Write-Host "`nBuilds info:"
     $builds | Format-Table -AutoSize -Property Name,Id,Status,Result,Link | Out-String -Width 10000
 
     # Return exit code based on status of builds
@@ -74,7 +122,7 @@ function Make-BuildsOutput {
         $failedBuilds | ForEach-Object -Process { Write-Host "##vso[task.logissue type=error;]Name: $($_.Name); Link: $($_.Link)" }
         Write-Host "##vso[task.complete result=Failed]"
     } else {
-        Write-host "##[section] All builds have been passed successfully"
+        Write-host "##[section]All builds have been passed successfully"
     }
 }
 
@@ -82,13 +130,22 @@ $azureDevOpsApi = Get-AzureDevOpsApi -TeamFoundationCollectionUri $TeamFoundatio
                                      -ProjectName $AzureDevOpsProjectName `
                                      -AccessToken $AzureDevOpsAccessToken
 
+$toolVersionsList = Get-ToolVersions -ManifestLink $ManifestLink `
+                                     -RetryIntervalSec $RetryIntervalSec `
+                                     -Retries $RetryCount `
+                                     -ToolVersions $ToolVersions
+
 $queuedBuilds = Queue-Builds -AzureDevOpsApi $azureDevOpsApi `
-                             -ToolVersions $ToolVersions `
+                             -ToolVersions $toolVersionsList `
                              -SourceBranch $SourceBranch `
                              -SourceVersion $SourceVersion `
-                             -DefinitionId $DefinitionId
-
-Write-Host "Waiting results of builds ..."
-Wait-Builds -Builds $queuedBuilds
+                             -DefinitionId $DefinitionId `
+                             -RetryIntervalSec $RetryIntervalSec `
+                             -Retries $RetryCount
 
-Make-BuildsOutput -Builds $queuedBuilds
+if ($WaitForBuilds) {
+    Write-Host "`nWaiting results of builds ..."
+    Wait-Builds -Builds $queuedBuilds -RetryIntervalSec $RetryIntervalSec
+    
+    Make-BuildsOutput -Builds $queuedBuilds
+}