Adding pypy notifications and optimizing workflow and parsers (#42)

.github/workflows/get-tools-new-versions.yml

@@ -9,74 +9,56 @@ defaults:
     shell: pwsh
 
 jobs:
-  find-new-xamarin-versions:
+  find-new-tool-versions:
-    name: Searching for new Xamarin versions
+    strategy:
+      fail-fast: false
+      matrix:
+        tool:
+          - name: 'Xamarin'
+            image: 'https://avatars.githubusercontent.com/u/790012?s=200&v=4'
+          - name: 'Python'
+            image: 'https://avatars.githubusercontent.com/u/1525981?s=200&v=4'
+          - name: 'PyPy'
+            image: 'https://avatars.githubusercontent.com/u/318667?s=200&v=4'
+    name: 'Searching for new versions of ${{ matrix.tool.name }}'
     runs-on: ubuntu-latest
-    outputs:
-      versions-output: ${{ steps.get-new-xamarin-versions.outputs.versions-output }}
     steps:
       - uses: actions/checkout@v2
-      - id: get-new-xamarin-versions
+      - id: get-new-tool-versions
-        name: Get new Xamarin versions
+        name: Get new tool versions
-        run: echo "::set-output name=versions-output::$(./get-new-tool-versions/verify-new-tool-version-added-to-image.ps1 -ToolName Xamarin)"
+        run: echo "::set-output name=versions-output::$(./get-new-tool-versions/verify-new-tool-version-added-to-image.ps1 -ToolName ${{ matrix.tool.name }})"
-  
+      - name: Check versions
-  check-new-xamarin-versions:
+        if: steps.get-new-tool-versions.outputs.versions-output == ''
-    name: Verifying new Xamarin versions exist
-    runs-on: ubuntu-latest
-    needs: find-new-xamarin-versions
-    steps:
-      - uses: actions/checkout@v2
-      - name: Check Versions
-        if: needs.find-new-xamarin-versions.outputs.versions-output == ''
-        run: Write-Host "No new versions found"
-      - uses: ./.github/actions/send-slack-notification
-        if: needs.find-new-xamarin-versions.outputs.versions-output != ''
-        with:
-          url: ${{ secrets.SLACK_CHANNEL_URL }}
-          tool-name: 'Xamarin'
-          tool-version: ${{ needs.find-new-xamarin-versions.outputs.versions-output }}
-          image-url: 'https://avatars.githubusercontent.com/u/790012?s=200&v=4'
-          add-to-toolset-flag: '-AddToToolsetFlag'
-  
-  find-new-python-versions:
-    name: Searching for new Python versions
-    runs-on: ubuntu-latest
-    outputs:
-      versions-output: ${{ steps.get-new-python-versions.outputs.versions-output }}
-    steps:
-      - uses: actions/checkout@v2
-      - id: get-new-python-versions
-        name: Get new Python versions
-        run: echo "::set-output name=versions-output::$(./get-new-tool-versions/verify-new-tool-version-added-to-image.ps1 -ToolName Python)"
-  
-  check-new-python-versions:
-    name: Verifying new Python versions exist
-    runs-on: ubuntu-latest
-    needs: find-new-python-versions
-    steps:
-      - uses: actions/checkout@v2
-      - name: Check Versions
-        if: needs.find-new-python-versions.outputs.versions-output == ''
         run: Write-Host "No new versions found"
       - uses: ./.github/actions/send-slack-notification
-        if: needs.find-new-python-versions.outputs.versions-output != ''
+        name: Send Slack notification
+        if: steps.get-new-tool-versions.outputs.versions-output != ''
         with:
           url: ${{ secrets.SLACK_CHANNEL_URL }}
-          tool-name: 'Python'
+          tool-name: '${{ matrix.tool.name }}'
-          tool-version: ${{ needs.find-new-python-versions.outputs.versions-output }}
+          tool-version: ${{ steps.get-new-tool-versions.outputs.versions-output }}
-          image-url: 'https://www.python.org/static/community_logos/python-powered-h-100x130.png'
+          image-url: '${{ matrix.tool.image }}'
           add-to-toolset-flag: '-AddToToolsetFlag'
 
   check_build:
     name: Check build for failures 
     runs-on: ubuntu-latest
-    needs: [check-new-xamarin-versions, check-new-python-versions]
+    needs: [find-new-tool-versions]
     if: failure()
     steps:
       - uses: actions/checkout@v2
+      - id: get-failed-jobs
+        name: Get failed jobs
+        run: |
+          $jobs_url = "$env:GITHUB_API_URL/repos/$env:GITHUB_REPOSITORY/actions/runs/$env:GITHUB_RUN_ID/jobs"
+          $failedJobs = (Invoke-RestMethod -Uri $jobs_url).jobs |
+            Where-Object conclusion -eq "failure" |
+            ForEach-Object {"\n\t" + $_.name.split(" ")[-1] + ": $($_.html_url)"}
+          echo "::set-output name=failed-jobs::$failedJobs"
       - uses: ./.github/actions/send-slack-notification
+        name: Send Slack notification about failure
         with:
           url: ${{ secrets.SLACK_CHANNEL_URL }}
-          tool-name: 'Python or Xamarin'
+          tool-name: 'Tool name'
           pipeline-url: '$env:GITHUB_SERVER_URL/$env:GITHUB_REPOSITORY/actions/runs/$env:GITHUB_RUN_ID'
-          text: "The build of the Xamarin or Python detection pipeline has failed stages:"
+          text: "Missing toolset tool versions checker pipeline has failed jobs:/n/t${{ steps.get-failed-jobs.outputs.failed-jobs }}"

azure-pipelines/get-tool-versions-xamarin.yml

@@ -1,37 +0,0 @@
-name: $(date:yyyyMMdd)$(rev:.r)
-trigger: none
-pr: none
-schedules:
-- cron: "0 8 * * Thu"
-  displayName: Daily build
-  branches:
-    include:
-    - main
-  always: true
-
-variables:
-  PoolName: 'Azure Pipelines'
-  VmImage: 'ubuntu-18.04'
-
-stages:
-- stage: Find_New_Versions
-  dependsOn: []
-  jobs:
-  - job: Find_New_Versions
-    pool:
-      name: $(PoolName)
-      vmImage: $(VmImage)
-    steps:
-    - template: /azure-pipelines/templates/get-tool-versions-steps.yml
-
-- stage: Check_New_Versions
-  dependsOn: Find_New_Versions
-  jobs:
-  - job: Check_New_Versions
-    pool:
-      name: $(PoolName)
-      vmImage: $(VmImage)
-    variables:
-      ToolVersions: $[ stageDependencies.Find_New_Versions.Find_New_Versions.outputs['Get_versions.TOOL_VERSIONS'] ]
-    steps:
-    - template: /azure-pipelines/templates/check-versions.yml

azure-pipelines/templates/check-versions.yml

@@ -21,7 +21,7 @@ steps:
     TargetType: inline
     script: |
       $ToolName = "$(TOOL_NAME)"
-      if ($ToolName -in @("Python", "Xamarin")) {
+      if ($ToolName -eq "Python") {
         $PipelineUrl = " "
       } else {
         $PipelineUrl = "$(System.TeamFoundationCollectionUri)$(System.TeamProject)/_build/results?buildId=$(Build.BuildId)"

get-new-tool-versions/get-new-tool-versions.ps1

@@ -3,7 +3,7 @@
 Check and return list of new available tool versions
 
 .PARAMETER ToolName
-Required parameter. The name of tool for which parser is available (Node, Go, Python, Xamarin)
+Required parameter. The name of tool for which parser is available (Node, Go, Python)
 #>
 
 param (
@@ -16,18 +16,10 @@ $ToolVersionParser = Get-ToolVersionsParser -ToolName $ToolName
 $VersionsFromDist = $ToolVersionParser.GetAvailableVersions()
 $VersionsFromManifest = $ToolVersionParser.GetUploadedVersions()
 
-$joinChars = ", "
+$VersionsToBuild = $VersionsFromDist | Where-Object { $VersionsFromManifest -notcontains $_ }
-if ($ToolName -eq "Xamarin") {
-    $VersionsToBuild = $VersionsFromDist | Where-Object { $VersionsFromManifest[$_.name] -notcontains $_.version } | ForEach-Object {[string]::Empty} {
-        '{0,-15} : {1}' -f $_.name, $_.version
-    }
-    $joinChars = "\n\t"
-} else {
-    $VersionsToBuild = $VersionsFromDist | Where-Object { $VersionsFromManifest -notcontains $_ }
-}
 
 if ($VersionsToBuild) {
-    $availableVersions = $VersionsToBuild -join $joinChars
+    $availableVersions = $VersionsToBuild -join ", "
     Write-Host "The following versions are available to build:`n${availableVersions}"
     Write-Host "::set-output name=TOOL_VERSIONS::${availableVersions}"
     Write-Host "##vso[task.setvariable variable=TOOL_VERSIONS;isOutput=true]${availableVersions}"

get-new-tool-versions/parsers/parsers-factory.psm1

@@ -1,7 +1,6 @@
 using module "./node-parser.psm1"
 using module "./go-parser.psm1"
 using module "./python-parser.psm1"
-using module "./xamarin-parser.psm1"
 
 function Get-ToolVersionsParser {
     param(
@@ -13,7 +12,6 @@ function Get-ToolVersionsParser {
         "Node" { return [NodeVersionsParser]::New() }
         "Go" { return [GoVersionsParser]::New() }
         "Python" { return [PythonVersionsParser]::New() }
-        "Xamarin" { return [XamarinversionsParser]::New() }
         Default {
             throw "Unknown tool name"
         }

get-new-tool-versions/parsers/verify-added-to-image/verify-python-parser.psm1

@@ -0,0 +1,21 @@
+function Search-PythonVersionsNotOnImage {
+    param (
+        [string]$ToolName,
+        [string]$ReleasesUrl,
+        [string]$FilterParameter,
+        [string]$FilterArch
+    )
+    
+    $stableReleases = (Invoke-RestMethod $ReleasesUrl) |
+        Where-Object stable -eq $true |
+        ForEach-Object {$_.$FilterParameter.split(".")[0,1] -join"."} |
+        Select-Object -Unique
+    $toolsetUrl = "https://raw.githubusercontent.com/actions/virtual-environments/main/images/win/toolsets/toolset-2022.json"
+    $latestVersion = ((Invoke-RestMethod $toolsetUrl).toolcache |
+        Where-Object {$_.name -eq $ToolName -and $_.arch -eq $FilterArch}).versions |
+        Select-Object -Last 1
+    $latestMinorVesion = $latestVersion.TrimEnd(".*")
+    $versionsToAdd = $stableReleases | Where-Object {[version]$_ -gt [version]$latestMinorVesion}
+    
+    return $versionsToAdd
+}

get-new-tool-versions/parsers/verify-added-to-image/verify-xamarin-parser.psm1

@@ -0,0 +1,19 @@
+function Search-XamarinVersionsNotOnImage {
+    param (
+        [string]$ReleasesUrl,
+        [array]$FilterProducts
+    )
+
+    $xamarinReleases = (Invoke-RestMethod $ReleasesUrl).items
+    $filteredReleases = $xamarinReleases | Where-Object {$_.name -in $FilterProducts.name} | Sort-Object name | Select-Object name, version
+    $toolsetUrl = "https://raw.githubusercontent.com/actions/virtual-environments/main/images/macos/toolsets/toolset-11.json"
+    $uploadedReleases = (Invoke-RestMethod $toolsetUrl).xamarin
+    $releasesOnImage = @()
+    foreach ($FilterProduct in $FilterProducts) {
+        $releasesOnImage += @{$FilterProduct.name = $uploadedReleases.($FilterProduct.property)}
+    }
+    $versionsToAdd = $filteredReleases | Where-Object {$releasesOnImage.($_.name) -notcontains $_.version} | ForEach-Object {[string]::Empty} {
+        '{0,-15} : {1}' -f $_.name, $_.version
+    }
+    return $versionsToAdd
+}

get-new-tool-versions/parsers/xamarin-parser.psm1

@@ -1,30 +0,0 @@
-using module "./base-parser.psm1"
-
-class XamarinVersionsParser: BaseVersionsParser {
-    [PSCustomObject] GetAvailableVersions() {
-        $allVersions = $this.ParseAllAvailableVersions()
-        return $allVersions
-    }
-
-    [hashtable] GetUploadedVersions() {
-        $url = $this.BuildGitHubFileUrl("actions", "virtual-environments", "main", "images/macos/toolsets/toolset-11.json")
-        $releases = Invoke-RestMethod $url -MaximumRetryCount $this.ApiRetryCount -RetryIntervalSec $this.ApiRetryIntervalSeconds
-        $xamarin = $releases.xamarin
-        $xamarinReleases = @{
-            'Mono Framework' = $xamarin.'mono-versions'
-            'Xamarin.Android' = $xamarin.'android-versions'
-            'Xamarin.iOS' = $xamarin.'ios-versions'
-            'Xamarin.Mac' = $xamarin.'mac-versions'
-        }
-        return $xamarinReleases
-    }
-
-    hidden [PSCustomObject] ParseAllAvailableVersions() {
-        $url = "http://aka.ms/manifest/stable"
-        $filteredProducts = @('Mono Framework', 'Xamarin.Android', 'Xamarin.iOS', 'Xamarin.Mac')
-        $releases = Invoke-RestMethod $url -MaximumRetryCount $this.ApiRetryCount -RetryIntervalSec $this.ApiRetryIntervalSeconds
-        $items = $releases.items
-        $products = $items | Where-Object {$_.name -in $filteredProducts} | Sort-Object name | Select-Object name, version
-        return $products
-    }
-}

get-new-tool-versions/verify-new-tool-version-added-to-image.ps1

@@ -3,46 +3,39 @@
 Check and return list of new available tool versions that not added to toolsets yet
 
 .PARAMETER ToolName
-Required parameter. The name of tool for which parser is available (Python, Xamarin)
+Required parameter. The name of tool for which parser is available (Python, Xamarin, PyPy)
 #>
 
 param (
     [Parameter(Mandatory)]
-    [ValidateSet("Python", "Xamarin")]
+    [ValidateSet("Python", "Xamarin", "PyPy")]
     [string]$ToolName
 )
 
+Get-ChildItem "$PSScriptRoot/parsers/verify-added-to-image/" | ForEach-Object {Import-Module $_.FullName}
+
 if ($ToolName -eq "Python") {
     $pythonVesionsManifestUrl = "https://raw.githubusercontent.com/actions/python-versions/main/versions-manifest.json"
-    $builtStableMinorVersions = (Invoke-RestMethod $pythonVesionsManifestUrl) |
+    $versionsToAdd = Search-PythonVersionsNotOnImage -ToolName $ToolName -ReleasesUrl $pythonVesionsManifestUrl -FilterParameter "version" -FilterArch "x64"
-        Where-Object stable -eq $true |
+}
-        ForEach-Object {$_.version.split(".")[0,1] -join"."} |
+
-        Select-Object -Unique
+if ($ToolName -eq "PyPy") {
-    $toolsetUrl = "https://raw.githubusercontent.com/actions/virtual-environments/main/images/win/toolsets/toolset-2019.json"
+    $pypyReleases = "https://downloads.python.org/pypy/versions.json"
-    $latestExistingMinorVesion = ((Invoke-RestMethod $toolsetUrl).toolcache |
+    $versionsToAdd = Search-PythonVersionsNotOnImage -ToolName $ToolName -ReleasesUrl $pypyReleases -FilterParameter "python_version" -FilterArch "x86"
-        Where-Object {$_.name -eq "Python" -and $_.arch -eq "x64"}).versions |
-        ForEach-Object {$_.split(".")[0,1] -join"."} |
-        Select-Object -Last 1
-    $versionsToAdd = $builtStableMinorVersions | Where-Object {[version]$_ -gt [version]$latestExistingMinorVesion}
 }
 
 if ($ToolName -eq "Xamarin") {
-    $xamarinReleases = (Invoke-RestMethod "http://aka.ms/manifest/stable").items
+    $xamarinReleases = "http://aka.ms/manifest/stable"
-    $xamarinProducts = @('Mono Framework', 'Xamarin.Android', 'Xamarin.iOS', 'Xamarin.Mac')
+    $xamarinProducts = @(
-    $filteredReleases = $xamarinReleases | Where-Object {$_.name -in $xamarinProducts} | Sort-Object name | Select-Object name, version
+        [PSCustomObject] @{name = 'Mono Framework'; property = 'mono-versions'}
-    $toolsetUrl = "https://raw.githubusercontent.com/actions/virtual-environments/main/images/macos/toolsets/toolset-11.json"
+        [PSCustomObject] @{name = 'Xamarin.Android'; property = 'android-versions'}
-    $uploadedReleases = (Invoke-RestMethod $toolsetUrl).xamarin
+        [PSCustomObject] @{name = 'Xamarin.iOS'; property = 'ios-versions'}
-    $releasesOnImage = @{
+        [PSCustomObject] @{name = 'Xamarin.Mac'; property = 'mac-versions'}
-        'Mono Framework' = $uploadedReleases.'mono-versions'
+    )
-        'Xamarin.Android' = $uploadedReleases.'android-versions'
+    $versionsToAdd = Search-XamarinVersionsNotOnImage -ReleasesUrl $xamarinReleases -FilterProducts $xamarinProducts
-        'Xamarin.iOS' = $uploadedReleases.'ios-versions'
-        'Xamarin.Mac' = $uploadedReleases.'mac-versions'
-    }
-    $versionsToAdd = $filteredReleases | Where-Object {$releasesOnImage[$_.name] -notcontains $_.version } | ForEach-Object {[string]::Empty} {
-        '{0,-15} : {1}' -f $_.name, $_.version
-    }
     $joinChars = "\n\t"
 }
+
 $versionsToAdd = $versionsToAdd -join $joinChars
 
 return $versionsToAdd