diff --git a/.github/workflows/get-tools-new-versions.yml b/.github/workflows/get-tools-new-versions.yml index 729ac15..ad06994 100644 --- a/.github/workflows/get-tools-new-versions.yml +++ b/.github/workflows/get-tools-new-versions.yml @@ -9,74 +9,56 @@ defaults: shell: pwsh jobs: - find-new-xamarin-versions: - name: Searching for new Xamarin versions + find-new-tool-versions: + strategy: + fail-fast: false + matrix: + tool: + - name: 'Xamarin' + image: 'https://avatars.githubusercontent.com/u/790012?s=200&v=4' + - name: 'Python' + image: 'https://avatars.githubusercontent.com/u/1525981?s=200&v=4' + - name: 'PyPy' + image: 'https://avatars.githubusercontent.com/u/318667?s=200&v=4' + name: 'Searching for new versions of ${{ matrix.tool.name }}' runs-on: ubuntu-latest - outputs: - versions-output: ${{ steps.get-new-xamarin-versions.outputs.versions-output }} steps: - uses: actions/checkout@v2 - - id: get-new-xamarin-versions - name: Get new Xamarin versions - run: echo "::set-output name=versions-output::$(./get-new-tool-versions/verify-new-tool-version-added-to-image.ps1 -ToolName Xamarin)" - - check-new-xamarin-versions: - name: Verifying new Xamarin versions exist - runs-on: ubuntu-latest - needs: find-new-xamarin-versions - steps: - - uses: actions/checkout@v2 - - name: Check Versions - if: needs.find-new-xamarin-versions.outputs.versions-output == '' - run: Write-Host "No new versions found" - - uses: ./.github/actions/send-slack-notification - if: needs.find-new-xamarin-versions.outputs.versions-output != '' - with: - url: ${{ secrets.SLACK_CHANNEL_URL }} - tool-name: 'Xamarin' - tool-version: ${{ needs.find-new-xamarin-versions.outputs.versions-output }} - image-url: 'https://avatars.githubusercontent.com/u/790012?s=200&v=4' - add-to-toolset-flag: '-AddToToolsetFlag' - - find-new-python-versions: - name: Searching for new Python versions - runs-on: ubuntu-latest - outputs: - versions-output: ${{ steps.get-new-python-versions.outputs.versions-output }} - steps: - - uses: actions/checkout@v2 - - id: get-new-python-versions - name: Get new Python versions - run: echo "::set-output name=versions-output::$(./get-new-tool-versions/verify-new-tool-version-added-to-image.ps1 -ToolName Python)" - - check-new-python-versions: - name: Verifying new Python versions exist - runs-on: ubuntu-latest - needs: find-new-python-versions - steps: - - uses: actions/checkout@v2 - - name: Check Versions - if: needs.find-new-python-versions.outputs.versions-output == '' + - id: get-new-tool-versions + name: Get new tool versions + run: echo "::set-output name=versions-output::$(./get-new-tool-versions/verify-new-tool-version-added-to-image.ps1 -ToolName ${{ matrix.tool.name }})" + - name: Check versions + if: steps.get-new-tool-versions.outputs.versions-output == '' run: Write-Host "No new versions found" - uses: ./.github/actions/send-slack-notification - if: needs.find-new-python-versions.outputs.versions-output != '' + name: Send Slack notification + if: steps.get-new-tool-versions.outputs.versions-output != '' with: url: ${{ secrets.SLACK_CHANNEL_URL }} - tool-name: 'Python' - tool-version: ${{ needs.find-new-python-versions.outputs.versions-output }} - image-url: 'https://www.python.org/static/community_logos/python-powered-h-100x130.png' + tool-name: '${{ matrix.tool.name }}' + tool-version: ${{ steps.get-new-tool-versions.outputs.versions-output }} + image-url: '${{ matrix.tool.image }}' add-to-toolset-flag: '-AddToToolsetFlag' check_build: name: Check build for failures runs-on: ubuntu-latest - needs: [check-new-xamarin-versions, check-new-python-versions] + needs: [find-new-tool-versions] if: failure() steps: - uses: actions/checkout@v2 + - id: get-failed-jobs + name: Get failed jobs + run: | + $jobs_url = "$env:GITHUB_API_URL/repos/$env:GITHUB_REPOSITORY/actions/runs/$env:GITHUB_RUN_ID/jobs" + $failedJobs = (Invoke-RestMethod -Uri $jobs_url).jobs | + Where-Object conclusion -eq "failure" | + ForEach-Object {"\n\t" + $_.name.split(" ")[-1] + ": $($_.html_url)"} + echo "::set-output name=failed-jobs::$failedJobs" - uses: ./.github/actions/send-slack-notification + name: Send Slack notification about failure with: url: ${{ secrets.SLACK_CHANNEL_URL }} - tool-name: 'Python or Xamarin' + tool-name: 'Tool name' pipeline-url: '$env:GITHUB_SERVER_URL/$env:GITHUB_REPOSITORY/actions/runs/$env:GITHUB_RUN_ID' - text: "The build of the Xamarin or Python detection pipeline has failed stages:" + text: "Missing toolset tool versions checker pipeline has failed jobs:/n/t${{ steps.get-failed-jobs.outputs.failed-jobs }}" \ No newline at end of file diff --git a/azure-pipelines/get-tool-versions-xamarin.yml b/azure-pipelines/get-tool-versions-xamarin.yml deleted file mode 100644 index 866f810..0000000 --- a/azure-pipelines/get-tool-versions-xamarin.yml +++ /dev/null @@ -1,37 +0,0 @@ -name: $(date:yyyyMMdd)$(rev:.r) -trigger: none -pr: none -schedules: -- cron: "0 8 * * Thu" - displayName: Daily build - branches: - include: - - main - always: true - -variables: - PoolName: 'Azure Pipelines' - VmImage: 'ubuntu-18.04' - -stages: -- stage: Find_New_Versions - dependsOn: [] - jobs: - - job: Find_New_Versions - pool: - name: $(PoolName) - vmImage: $(VmImage) - steps: - - template: /azure-pipelines/templates/get-tool-versions-steps.yml - -- stage: Check_New_Versions - dependsOn: Find_New_Versions - jobs: - - job: Check_New_Versions - pool: - name: $(PoolName) - vmImage: $(VmImage) - variables: - ToolVersions: $[ stageDependencies.Find_New_Versions.Find_New_Versions.outputs['Get_versions.TOOL_VERSIONS'] ] - steps: - - template: /azure-pipelines/templates/check-versions.yml diff --git a/azure-pipelines/templates/check-versions.yml b/azure-pipelines/templates/check-versions.yml index 9af96ef..084c913 100644 --- a/azure-pipelines/templates/check-versions.yml +++ b/azure-pipelines/templates/check-versions.yml @@ -21,7 +21,7 @@ steps: TargetType: inline script: | $ToolName = "$(TOOL_NAME)" - if ($ToolName -in @("Python", "Xamarin")) { + if ($ToolName -eq "Python") { $PipelineUrl = " " } else { $PipelineUrl = "$(System.TeamFoundationCollectionUri)$(System.TeamProject)/_build/results?buildId=$(Build.BuildId)" diff --git a/get-new-tool-versions/get-new-tool-versions.ps1 b/get-new-tool-versions/get-new-tool-versions.ps1 index 558dd0f..e8342ed 100644 --- a/get-new-tool-versions/get-new-tool-versions.ps1 +++ b/get-new-tool-versions/get-new-tool-versions.ps1 @@ -3,7 +3,7 @@ Check and return list of new available tool versions .PARAMETER ToolName -Required parameter. The name of tool for which parser is available (Node, Go, Python, Xamarin) +Required parameter. The name of tool for which parser is available (Node, Go, Python) #> param ( @@ -16,18 +16,10 @@ $ToolVersionParser = Get-ToolVersionsParser -ToolName $ToolName $VersionsFromDist = $ToolVersionParser.GetAvailableVersions() $VersionsFromManifest = $ToolVersionParser.GetUploadedVersions() -$joinChars = ", " -if ($ToolName -eq "Xamarin") { - $VersionsToBuild = $VersionsFromDist | Where-Object { $VersionsFromManifest[$_.name] -notcontains $_.version } | ForEach-Object {[string]::Empty} { - '{0,-15} : {1}' -f $_.name, $_.version - } - $joinChars = "\n\t" -} else { - $VersionsToBuild = $VersionsFromDist | Where-Object { $VersionsFromManifest -notcontains $_ } -} +$VersionsToBuild = $VersionsFromDist | Where-Object { $VersionsFromManifest -notcontains $_ } if ($VersionsToBuild) { - $availableVersions = $VersionsToBuild -join $joinChars + $availableVersions = $VersionsToBuild -join ", " Write-Host "The following versions are available to build:`n${availableVersions}" Write-Host "::set-output name=TOOL_VERSIONS::${availableVersions}" Write-Host "##vso[task.setvariable variable=TOOL_VERSIONS;isOutput=true]${availableVersions}" diff --git a/get-new-tool-versions/parsers/parsers-factory.psm1 b/get-new-tool-versions/parsers/parsers-factory.psm1 index 199136c..eb9b69c 100644 --- a/get-new-tool-versions/parsers/parsers-factory.psm1 +++ b/get-new-tool-versions/parsers/parsers-factory.psm1 @@ -1,7 +1,6 @@ using module "./node-parser.psm1" using module "./go-parser.psm1" using module "./python-parser.psm1" -using module "./xamarin-parser.psm1" function Get-ToolVersionsParser { param( @@ -13,7 +12,6 @@ function Get-ToolVersionsParser { "Node" { return [NodeVersionsParser]::New() } "Go" { return [GoVersionsParser]::New() } "Python" { return [PythonVersionsParser]::New() } - "Xamarin" { return [XamarinversionsParser]::New() } Default { throw "Unknown tool name" } diff --git a/get-new-tool-versions/parsers/verify-added-to-image/verify-python-parser.psm1 b/get-new-tool-versions/parsers/verify-added-to-image/verify-python-parser.psm1 new file mode 100644 index 0000000..d8c6a55 --- /dev/null +++ b/get-new-tool-versions/parsers/verify-added-to-image/verify-python-parser.psm1 @@ -0,0 +1,21 @@ +function Search-PythonVersionsNotOnImage { + param ( + [string]$ToolName, + [string]$ReleasesUrl, + [string]$FilterParameter, + [string]$FilterArch + ) + + $stableReleases = (Invoke-RestMethod $ReleasesUrl) | + Where-Object stable -eq $true | + ForEach-Object {$_.$FilterParameter.split(".")[0,1] -join"."} | + Select-Object -Unique + $toolsetUrl = "https://raw.githubusercontent.com/actions/virtual-environments/main/images/win/toolsets/toolset-2022.json" + $latestVersion = ((Invoke-RestMethod $toolsetUrl).toolcache | + Where-Object {$_.name -eq $ToolName -and $_.arch -eq $FilterArch}).versions | + Select-Object -Last 1 + $latestMinorVesion = $latestVersion.TrimEnd(".*") + $versionsToAdd = $stableReleases | Where-Object {[version]$_ -gt [version]$latestMinorVesion} + + return $versionsToAdd +} \ No newline at end of file diff --git a/get-new-tool-versions/parsers/verify-added-to-image/verify-xamarin-parser.psm1 b/get-new-tool-versions/parsers/verify-added-to-image/verify-xamarin-parser.psm1 new file mode 100644 index 0000000..4999217 --- /dev/null +++ b/get-new-tool-versions/parsers/verify-added-to-image/verify-xamarin-parser.psm1 @@ -0,0 +1,19 @@ +function Search-XamarinVersionsNotOnImage { + param ( + [string]$ReleasesUrl, + [array]$FilterProducts + ) + + $xamarinReleases = (Invoke-RestMethod $ReleasesUrl).items + $filteredReleases = $xamarinReleases | Where-Object {$_.name -in $FilterProducts.name} | Sort-Object name | Select-Object name, version + $toolsetUrl = "https://raw.githubusercontent.com/actions/virtual-environments/main/images/macos/toolsets/toolset-11.json" + $uploadedReleases = (Invoke-RestMethod $toolsetUrl).xamarin + $releasesOnImage = @() + foreach ($FilterProduct in $FilterProducts) { + $releasesOnImage += @{$FilterProduct.name = $uploadedReleases.($FilterProduct.property)} + } + $versionsToAdd = $filteredReleases | Where-Object {$releasesOnImage.($_.name) -notcontains $_.version} | ForEach-Object {[string]::Empty} { + '{0,-15} : {1}' -f $_.name, $_.version + } + return $versionsToAdd +} \ No newline at end of file diff --git a/get-new-tool-versions/parsers/xamarin-parser.psm1 b/get-new-tool-versions/parsers/xamarin-parser.psm1 deleted file mode 100644 index fb5c135..0000000 --- a/get-new-tool-versions/parsers/xamarin-parser.psm1 +++ /dev/null @@ -1,30 +0,0 @@ -using module "./base-parser.psm1" - -class XamarinVersionsParser: BaseVersionsParser { - [PSCustomObject] GetAvailableVersions() { - $allVersions = $this.ParseAllAvailableVersions() - return $allVersions - } - - [hashtable] GetUploadedVersions() { - $url = $this.BuildGitHubFileUrl("actions", "virtual-environments", "main", "images/macos/toolsets/toolset-11.json") - $releases = Invoke-RestMethod $url -MaximumRetryCount $this.ApiRetryCount -RetryIntervalSec $this.ApiRetryIntervalSeconds - $xamarin = $releases.xamarin - $xamarinReleases = @{ - 'Mono Framework' = $xamarin.'mono-versions' - 'Xamarin.Android' = $xamarin.'android-versions' - 'Xamarin.iOS' = $xamarin.'ios-versions' - 'Xamarin.Mac' = $xamarin.'mac-versions' - } - return $xamarinReleases - } - - hidden [PSCustomObject] ParseAllAvailableVersions() { - $url = "http://aka.ms/manifest/stable" - $filteredProducts = @('Mono Framework', 'Xamarin.Android', 'Xamarin.iOS', 'Xamarin.Mac') - $releases = Invoke-RestMethod $url -MaximumRetryCount $this.ApiRetryCount -RetryIntervalSec $this.ApiRetryIntervalSeconds - $items = $releases.items - $products = $items | Where-Object {$_.name -in $filteredProducts} | Sort-Object name | Select-Object name, version - return $products - } -} \ No newline at end of file diff --git a/get-new-tool-versions/verify-new-tool-version-added-to-image.ps1 b/get-new-tool-versions/verify-new-tool-version-added-to-image.ps1 index da09401..c4b088c 100644 --- a/get-new-tool-versions/verify-new-tool-version-added-to-image.ps1 +++ b/get-new-tool-versions/verify-new-tool-version-added-to-image.ps1 @@ -3,46 +3,39 @@ Check and return list of new available tool versions that not added to toolsets yet .PARAMETER ToolName -Required parameter. The name of tool for which parser is available (Python, Xamarin) +Required parameter. The name of tool for which parser is available (Python, Xamarin, PyPy) #> param ( [Parameter(Mandatory)] - [ValidateSet("Python", "Xamarin")] + [ValidateSet("Python", "Xamarin", "PyPy")] [string]$ToolName ) +Get-ChildItem "$PSScriptRoot/parsers/verify-added-to-image/" | ForEach-Object {Import-Module $_.FullName} + if ($ToolName -eq "Python") { $pythonVesionsManifestUrl = "https://raw.githubusercontent.com/actions/python-versions/main/versions-manifest.json" - $builtStableMinorVersions = (Invoke-RestMethod $pythonVesionsManifestUrl) | - Where-Object stable -eq $true | - ForEach-Object {$_.version.split(".")[0,1] -join"."} | - Select-Object -Unique - $toolsetUrl = "https://raw.githubusercontent.com/actions/virtual-environments/main/images/win/toolsets/toolset-2019.json" - $latestExistingMinorVesion = ((Invoke-RestMethod $toolsetUrl).toolcache | - Where-Object {$_.name -eq "Python" -and $_.arch -eq "x64"}).versions | - ForEach-Object {$_.split(".")[0,1] -join"."} | - Select-Object -Last 1 - $versionsToAdd = $builtStableMinorVersions | Where-Object {[version]$_ -gt [version]$latestExistingMinorVesion} + $versionsToAdd = Search-PythonVersionsNotOnImage -ToolName $ToolName -ReleasesUrl $pythonVesionsManifestUrl -FilterParameter "version" -FilterArch "x64" +} + +if ($ToolName -eq "PyPy") { + $pypyReleases = "https://downloads.python.org/pypy/versions.json" + $versionsToAdd = Search-PythonVersionsNotOnImage -ToolName $ToolName -ReleasesUrl $pypyReleases -FilterParameter "python_version" -FilterArch "x86" } if ($ToolName -eq "Xamarin") { - $xamarinReleases = (Invoke-RestMethod "http://aka.ms/manifest/stable").items - $xamarinProducts = @('Mono Framework', 'Xamarin.Android', 'Xamarin.iOS', 'Xamarin.Mac') - $filteredReleases = $xamarinReleases | Where-Object {$_.name -in $xamarinProducts} | Sort-Object name | Select-Object name, version - $toolsetUrl = "https://raw.githubusercontent.com/actions/virtual-environments/main/images/macos/toolsets/toolset-11.json" - $uploadedReleases = (Invoke-RestMethod $toolsetUrl).xamarin - $releasesOnImage = @{ - 'Mono Framework' = $uploadedReleases.'mono-versions' - 'Xamarin.Android' = $uploadedReleases.'android-versions' - 'Xamarin.iOS' = $uploadedReleases.'ios-versions' - 'Xamarin.Mac' = $uploadedReleases.'mac-versions' - } - $versionsToAdd = $filteredReleases | Where-Object {$releasesOnImage[$_.name] -notcontains $_.version } | ForEach-Object {[string]::Empty} { - '{0,-15} : {1}' -f $_.name, $_.version - } + $xamarinReleases = "http://aka.ms/manifest/stable" + $xamarinProducts = @( + [PSCustomObject] @{name = 'Mono Framework'; property = 'mono-versions'} + [PSCustomObject] @{name = 'Xamarin.Android'; property = 'android-versions'} + [PSCustomObject] @{name = 'Xamarin.iOS'; property = 'ios-versions'} + [PSCustomObject] @{name = 'Xamarin.Mac'; property = 'mac-versions'} + ) + $versionsToAdd = Search-XamarinVersionsNotOnImage -ReleasesUrl $xamarinReleases -FilterProducts $xamarinProducts $joinChars = "\n\t" } + $versionsToAdd = $versionsToAdd -join $joinChars return $versionsToAdd