From 5cbb4af9a1cda5c3952ca873b3b0e38200dc2de7 Mon Sep 17 00:00:00 2001 From: Maksim Shilov Date: Thu, 28 Oct 2021 11:24:42 +0300 Subject: [PATCH] Adding pypy notifications and optimizing workflow and parsers --- .github/workflows/get-tools-new-versions.yml | 74 ++++++------------- .../verify-python-parser.psm1 | 21 ++++++ .../verify-xamarin-parser.psm1 | 19 +++++ ...verify-new-tool-version-added-to-image.ps1 | 45 +++++------ 4 files changed, 82 insertions(+), 77 deletions(-) create mode 100644 get-new-tool-versions/parsers/verify-added-to-image/verify-python-parser.psm1 create mode 100644 get-new-tool-versions/parsers/verify-added-to-image/verify-xamarin-parser.psm1 diff --git a/.github/workflows/get-tools-new-versions.yml b/.github/workflows/get-tools-new-versions.yml index 729ac15..569ae93 100644 --- a/.github/workflows/get-tools-new-versions.yml +++ b/.github/workflows/get-tools-new-versions.yml @@ -9,68 +9,40 @@ defaults: shell: pwsh jobs: - find-new-xamarin-versions: - name: Searching for new Xamarin versions + find-new-tool-versions: + strategy: + matrix: + tool: + - name: 'Xamarin' + image: 'https://avatars.githubusercontent.com/u/790012?s=200&v=4' + - name: 'Python' + image: 'https://avatars.githubusercontent.com/u/1525981?s=200&v=4' + - name: 'PyPy' + image: 'https://avatars.githubusercontent.com/u/318667?s=200&v=4' + name: Searching for new tool versions runs-on: ubuntu-latest - outputs: - versions-output: ${{ steps.get-new-xamarin-versions.outputs.versions-output }} steps: - uses: actions/checkout@v2 - - id: get-new-xamarin-versions - name: Get new Xamarin versions - run: echo "::set-output name=versions-output::$(./get-new-tool-versions/verify-new-tool-version-added-to-image.ps1 -ToolName Xamarin)" - - check-new-xamarin-versions: - name: Verifying new Xamarin versions exist - runs-on: ubuntu-latest - needs: find-new-xamarin-versions - steps: - - uses: actions/checkout@v2 - - name: Check Versions - if: needs.find-new-xamarin-versions.outputs.versions-output == '' - run: Write-Host "No new versions found" - - uses: ./.github/actions/send-slack-notification - if: needs.find-new-xamarin-versions.outputs.versions-output != '' - with: - url: ${{ secrets.SLACK_CHANNEL_URL }} - tool-name: 'Xamarin' - tool-version: ${{ needs.find-new-xamarin-versions.outputs.versions-output }} - image-url: 'https://avatars.githubusercontent.com/u/790012?s=200&v=4' - add-to-toolset-flag: '-AddToToolsetFlag' - - find-new-python-versions: - name: Searching for new Python versions - runs-on: ubuntu-latest - outputs: - versions-output: ${{ steps.get-new-python-versions.outputs.versions-output }} - steps: - - uses: actions/checkout@v2 - - id: get-new-python-versions - name: Get new Python versions - run: echo "::set-output name=versions-output::$(./get-new-tool-versions/verify-new-tool-version-added-to-image.ps1 -ToolName Python)" - - check-new-python-versions: - name: Verifying new Python versions exist - runs-on: ubuntu-latest - needs: find-new-python-versions - steps: - - uses: actions/checkout@v2 - - name: Check Versions - if: needs.find-new-python-versions.outputs.versions-output == '' + - id: get-new-tool-versions + name: Get new tool versions + run: echo "::set-output name=versions-output::$(./get-new-tool-versions/verify-new-tool-version-added-to-image.ps1 -ToolName ${{ matrix.tool.name }})" + - name: Check versions + if: steps.get-new-tool-versions.outputs.versions-output == '' run: Write-Host "No new versions found" - uses: ./.github/actions/send-slack-notification - if: needs.find-new-python-versions.outputs.versions-output != '' + name: Send Slack notification + if: steps.get-new-tool-versions.outputs.versions-output != '' with: url: ${{ secrets.SLACK_CHANNEL_URL }} - tool-name: 'Python' - tool-version: ${{ needs.find-new-python-versions.outputs.versions-output }} - image-url: 'https://www.python.org/static/community_logos/python-powered-h-100x130.png' + tool-name: '${{ matrix.tool.name }}' + tool-version: ${{ steps.get-new-tool-versions.outputs.versions-output }} + image-url: '${{ matrix.tool.image }}' add-to-toolset-flag: '-AddToToolsetFlag' check_build: name: Check build for failures runs-on: ubuntu-latest - needs: [check-new-xamarin-versions, check-new-python-versions] + needs: [find-new-tool-versions] if: failure() steps: - uses: actions/checkout@v2 @@ -79,4 +51,4 @@ jobs: url: ${{ secrets.SLACK_CHANNEL_URL }} tool-name: 'Python or Xamarin' pipeline-url: '$env:GITHUB_SERVER_URL/$env:GITHUB_REPOSITORY/actions/runs/$env:GITHUB_RUN_ID' - text: "The build of the Xamarin or Python detection pipeline has failed stages:" + text: "The build of the Xamarin or Python detection pipeline has failed stages:" \ No newline at end of file diff --git a/get-new-tool-versions/parsers/verify-added-to-image/verify-python-parser.psm1 b/get-new-tool-versions/parsers/verify-added-to-image/verify-python-parser.psm1 new file mode 100644 index 0000000..8eb10d5 --- /dev/null +++ b/get-new-tool-versions/parsers/verify-added-to-image/verify-python-parser.psm1 @@ -0,0 +1,21 @@ +function Search-PythonVersionsNotOnImage { + param ( + [string]$ToolName, + [string]$ReleasesUrl, + [string]$FilterParameter, + [string]$FilterArch + ) + + $stableReleases = (Invoke-RestMethod $ReleasesUrl) | + Where-Object stable -eq $true | + ForEach-Object {$_.$FilterParameter.split(".")[0,1] -join"."} | + Select-Object -Unique + $toolsetUrl = "https://raw.githubusercontent.com/actions/virtual-environments/main/images/win/toolsets/toolset-2019.json" + $latestExistingMinorVesion = ((Invoke-RestMethod $toolsetUrl).toolcache | + Where-Object {$_.name -eq $ToolName -and $_.arch -eq $FilterArch}).versions | + ForEach-Object {$_.split(".")[0,1] -join"."} | + Select-Object -Last 1 + $versionsToAdd = $stableReleases | Where-Object {[version]$_ -gt [version]$latestExistingMinorVesion} + + return $versionsToAdd +} \ No newline at end of file diff --git a/get-new-tool-versions/parsers/verify-added-to-image/verify-xamarin-parser.psm1 b/get-new-tool-versions/parsers/verify-added-to-image/verify-xamarin-parser.psm1 new file mode 100644 index 0000000..a286937 --- /dev/null +++ b/get-new-tool-versions/parsers/verify-added-to-image/verify-xamarin-parser.psm1 @@ -0,0 +1,19 @@ +function Search-XamarinVersionsNotOnImage { + param ( + [string]$ReleasesUrl, + [array]$FilterProducts + ) + + $xamarinReleases = (Invoke-RestMethod $ReleasesUrl).items + $filteredReleases = $xamarinReleases | Where-Object {$_.name -in $FilterProducts.name} | Sort-Object name | Select-Object name, version + $toolsetUrl = "https://raw.githubusercontent.com/actions/virtual-environments/main/images/macos/toolsets/toolset-11.json" + $uploadedReleases = (Invoke-RestMethod $toolsetUrl).xamarin + $releasesOnImage = @() + foreach ($FilterProduct in $FilterProducts) { + $releasesOnImage += @{$FilterProduct.name = $uploadedReleases.($FilterProduct.property)} + } + $versionsToAdd = $filteredReleases | Where-Object {$releasesOnImage.($_.name) -notcontains $_.version } | ForEach-Object {[string]::Empty} { + '{0,-15} : {1}' -f $_.name, $_.version + } + return $versionsToAdd +} \ No newline at end of file diff --git a/get-new-tool-versions/verify-new-tool-version-added-to-image.ps1 b/get-new-tool-versions/verify-new-tool-version-added-to-image.ps1 index da09401..c4b088c 100644 --- a/get-new-tool-versions/verify-new-tool-version-added-to-image.ps1 +++ b/get-new-tool-versions/verify-new-tool-version-added-to-image.ps1 @@ -3,46 +3,39 @@ Check and return list of new available tool versions that not added to toolsets yet .PARAMETER ToolName -Required parameter. The name of tool for which parser is available (Python, Xamarin) +Required parameter. The name of tool for which parser is available (Python, Xamarin, PyPy) #> param ( [Parameter(Mandatory)] - [ValidateSet("Python", "Xamarin")] + [ValidateSet("Python", "Xamarin", "PyPy")] [string]$ToolName ) +Get-ChildItem "$PSScriptRoot/parsers/verify-added-to-image/" | ForEach-Object {Import-Module $_.FullName} + if ($ToolName -eq "Python") { $pythonVesionsManifestUrl = "https://raw.githubusercontent.com/actions/python-versions/main/versions-manifest.json" - $builtStableMinorVersions = (Invoke-RestMethod $pythonVesionsManifestUrl) | - Where-Object stable -eq $true | - ForEach-Object {$_.version.split(".")[0,1] -join"."} | - Select-Object -Unique - $toolsetUrl = "https://raw.githubusercontent.com/actions/virtual-environments/main/images/win/toolsets/toolset-2019.json" - $latestExistingMinorVesion = ((Invoke-RestMethod $toolsetUrl).toolcache | - Where-Object {$_.name -eq "Python" -and $_.arch -eq "x64"}).versions | - ForEach-Object {$_.split(".")[0,1] -join"."} | - Select-Object -Last 1 - $versionsToAdd = $builtStableMinorVersions | Where-Object {[version]$_ -gt [version]$latestExistingMinorVesion} + $versionsToAdd = Search-PythonVersionsNotOnImage -ToolName $ToolName -ReleasesUrl $pythonVesionsManifestUrl -FilterParameter "version" -FilterArch "x64" +} + +if ($ToolName -eq "PyPy") { + $pypyReleases = "https://downloads.python.org/pypy/versions.json" + $versionsToAdd = Search-PythonVersionsNotOnImage -ToolName $ToolName -ReleasesUrl $pypyReleases -FilterParameter "python_version" -FilterArch "x86" } if ($ToolName -eq "Xamarin") { - $xamarinReleases = (Invoke-RestMethod "http://aka.ms/manifest/stable").items - $xamarinProducts = @('Mono Framework', 'Xamarin.Android', 'Xamarin.iOS', 'Xamarin.Mac') - $filteredReleases = $xamarinReleases | Where-Object {$_.name -in $xamarinProducts} | Sort-Object name | Select-Object name, version - $toolsetUrl = "https://raw.githubusercontent.com/actions/virtual-environments/main/images/macos/toolsets/toolset-11.json" - $uploadedReleases = (Invoke-RestMethod $toolsetUrl).xamarin - $releasesOnImage = @{ - 'Mono Framework' = $uploadedReleases.'mono-versions' - 'Xamarin.Android' = $uploadedReleases.'android-versions' - 'Xamarin.iOS' = $uploadedReleases.'ios-versions' - 'Xamarin.Mac' = $uploadedReleases.'mac-versions' - } - $versionsToAdd = $filteredReleases | Where-Object {$releasesOnImage[$_.name] -notcontains $_.version } | ForEach-Object {[string]::Empty} { - '{0,-15} : {1}' -f $_.name, $_.version - } + $xamarinReleases = "http://aka.ms/manifest/stable" + $xamarinProducts = @( + [PSCustomObject] @{name = 'Mono Framework'; property = 'mono-versions'} + [PSCustomObject] @{name = 'Xamarin.Android'; property = 'android-versions'} + [PSCustomObject] @{name = 'Xamarin.iOS'; property = 'ios-versions'} + [PSCustomObject] @{name = 'Xamarin.Mac'; property = 'mac-versions'} + ) + $versionsToAdd = Search-XamarinVersionsNotOnImage -ReleasesUrl $xamarinReleases -FilterProducts $xamarinProducts $joinChars = "\n\t" } + $versionsToAdd = $versionsToAdd -join $joinChars return $versionsToAdd