From be051a1f12cf72831a794ca2ec3a3e61c76b7e03 Mon Sep 17 00:00:00 2001 From: Maxim Lobanov Date: Wed, 9 Dec 2020 11:06:04 +0300 Subject: [PATCH] add parsers --- .../templates/get-tool-versions-steps.yml | 5 +- .../get-new-tool-versions.ps1 | 61 ++++--------------- get-new-tool-versions/helpers.psm1 | 14 ----- .../parsers/base-parser.psm1 | 32 ++++++++++ get-new-tool-versions/parsers/go-parser.psm1 | 25 ++++++++ .../parsers/node-parser.psm1 | 30 +++++++++ .../parsers/parsers-factory.psm1 | 19 ++++++ .../parsers/python-parser.psm1 | 54 ++++++++++++++++ 8 files changed, 173 insertions(+), 67 deletions(-) create mode 100644 get-new-tool-versions/parsers/base-parser.psm1 create mode 100644 get-new-tool-versions/parsers/go-parser.psm1 create mode 100644 get-new-tool-versions/parsers/node-parser.psm1 create mode 100644 get-new-tool-versions/parsers/parsers-factory.psm1 create mode 100644 get-new-tool-versions/parsers/python-parser.psm1 diff --git a/azure-pipelines/templates/get-tool-versions-steps.yml b/azure-pipelines/templates/get-tool-versions-steps.yml index 4dae6fc..e01a238 100644 --- a/azure-pipelines/templates/get-tool-versions-steps.yml +++ b/azure-pipelines/templates/get-tool-versions-steps.yml @@ -6,10 +6,7 @@ steps: targetType: filePath filePath: './get-new-tool-versions/get-new-tool-versions.ps1' arguments: | - -DistURL "$(DIST_URL)" ` - -ManifestLink "$(MANIFEST_URL)" ` - -VersionFilterToInclude $(INCLUDE_FILTER) ` - -VersionFilterToExclude $(EXCLUDE_FILTER) + -ToolName "$(TOOL_NAME)" - task: PowerShell@2 displayName: 'Cancel build' diff --git a/get-new-tool-versions/get-new-tool-versions.ps1 b/get-new-tool-versions/get-new-tool-versions.ps1 index af4b1a8..2e9064d 100644 --- a/get-new-tool-versions/get-new-tool-versions.ps1 +++ b/get-new-tool-versions/get-new-tool-versions.ps1 @@ -17,61 +17,24 @@ Optional parameter. Retry count #> param ( - [Parameter(Mandatory)] [string] $DistURL, - [Parameter(Mandatory)] [string] $ManifestLink, - [string[]] $VersionFilterToInclude, - [string[]] $VersionFilterToExclude, - [UInt32] $RetryIntervalSec = 60, - [UInt32] $RetryCount = 3 + [Parameter(Mandatory)] [string] $ToolName ) -Import-Module (Join-Path $PSScriptRoot "helpers.psm1") +Import-Module "$PSScriptRoot/parsers/parsers-factory.psm1" -function Get-VersionsByUrl { - param ( - [Parameter(Mandatory)] [string] $ToolPackagesUrl, - [Parameter(Mandatory)] [UInt32] $RetryIntervalSec, - [Parameter(Mandatory)] [UInt32] $RetryCount - ) +$ToolVersionParser = Get-ToolVersionsParser -ToolName $ToolName +$VersionsFromDist = $ToolVersionParser.GetAvailableVersions() +$VersionsFromManifest = $ToolVersionParser.GetUploadedVersions() - $packages = Invoke-RestMethod $ToolPackagesUrl -MaximumRetryCount $RetryCount -RetryIntervalSec $RetryIntervalSec - return $packages.version -} - -if ($VersionFilterToInclude) { - Validate-FiltersFormat -Filters $VersionFilterToInclude -} - -if ($VersionFilterToExclude) { - Validate-FiltersFormat -Filters $VersionFilterToExclude -} - -Write-Host "Get the packages list from $DistURL" -$versionsFromDist = Get-VersionsByUrl -ToolPackagesUrl $DistURL ` - -RetryIntervalSec $RetryIntervalSec ` - -RetryCount $RetryCount - -Write-Host "Get the packages list from $ManifestLink" -[Version[]] $versionsFromManifest = Get-VersionsByUrl -ToolPackagesUrl $ManifestLink ` - -RetryIntervalSec $RetryIntervalSec ` - -RetryCount $RetryCount - -[Version[]] $formattedVersions = Format-Versions -Versions $versionsFromDist - -$formattedVersions = Select-VersionsByFilter -Versions $formattedVersions ` - -IncludeFilters $VersionFilterToInclude ` - -ExcludeFilters $VersionFilterToExclude - -if (-not $formattedVersions) { - Write-Host "Couldn't find available versions with current filters" - exit 1 -} +Write-Host "Dist" +$VersionsFromDist | ForEach-Object { Write-Host $_ } +Write-Host "Manifest" +$VersionsFromManifest | ForEach-Object { Write-Host $_ } -$versionsToBuild = Skip-ExistingVersions -VersionsFromManifest $versionsFromManifest ` - -VersionsFromDist $formattedVersions +$VersionsToBuild = $VersionsFromDist | Where-Object { $VersionsFromManifest -notcontains $_ } -if ($versionsToBuild) { - $availableVersions = $versionsToBuild -join "," +if ($VersionsToBuild) { + $availableVersions = $VersionsToBuild -join "," $toolVersions = $availableVersions.Replace(",",", ") Write-Host "The following versions are available to build:`n$toolVersions" Write-Output "##vso[task.setvariable variable=TOOL_VERSIONS;isOutput=true]$toolVersions" diff --git a/get-new-tool-versions/helpers.psm1 b/get-new-tool-versions/helpers.psm1 index fb63f61..488d6c1 100644 --- a/get-new-tool-versions/helpers.psm1 +++ b/get-new-tool-versions/helpers.psm1 @@ -1,17 +1,3 @@ -function Validate-FiltersFormat { - param ( - [Parameter(Mandatory)] [string[]] $Filters - ) - - foreach($filter in $Filters) { - $filter.Split('.') | ForEach-Object { - if (($_ -notmatch '^\d+$') -and ($_ -ne '*')) { - throw "Invalid filter format - $filter" - } - } - } -} - function Format-Versions { param ( [Parameter(Mandatory)] [string[]] $Versions diff --git a/get-new-tool-versions/parsers/base-parser.psm1 b/get-new-tool-versions/parsers/base-parser.psm1 new file mode 100644 index 0000000..706f879 --- /dev/null +++ b/get-new-tool-versions/parsers/base-parser.psm1 @@ -0,0 +1,32 @@ +class BaseVersionsParser { + [Int32]$ApiRetryCount = 3 + [Int32]$ApiRetryIntervalSeconds = 60 + + [SemVer[]] GetAvailableVersions() { + $allVersionsRaw = $this.ParseAllAvailableVersions() + $allVersions = $allVersionsRaw | ForEach-Object { Write-Host $_; $this.FormatVersion($_) } + $filteredVersions = $allVersions | Where-Object { $this.ShouldIncludeVersion($_) } + return $filteredVersions + } + + [SemVer[]] GetUploadedVersions() { + throw "Method is not implemented in base class" + } + + hidden [SemVer[]] ParseAllAvailableVersions() { + throw "Method is not implemented in base class" + } + + hidden [SemVer] FormatVersion([string]$VersionSpec) { + throw "Method is not implemented in base class" + } + + hidden [bool] ShouldIncludeVersion([SemVer]$Version) { + throw "Method is not implemented in base class" + } + + hidden [string] BuildGitHubFileUrl($OrganizationName, $RepositoryName, $BranchName, $FilePath) { + # https://raw.githubusercontent.com/actions/node-versions/main/versions-manifest.json + return "https://raw.githubusercontent.com/${OrganizationName}/${RepositoryName}/${BranchName}/${FilePath}" + } +} \ No newline at end of file diff --git a/get-new-tool-versions/parsers/go-parser.psm1 b/get-new-tool-versions/parsers/go-parser.psm1 new file mode 100644 index 0000000..6091555 --- /dev/null +++ b/get-new-tool-versions/parsers/go-parser.psm1 @@ -0,0 +1,25 @@ +using module "./base-parser.psm1" + +class GoVersionsParser: BaseVersionsParser { + [SemVer[]] GetUploadedVersions() { + $url = $this.BuildGitHubFileUrl("actions", "go-versions", "main", "versions-manifest.json") + $releases = Invoke-RestMethod $url -MaximumRetryCount $this.ApiRetryCount -RetryIntervalSec $this.ApiRetryIntervalSeconds + return $releases.version + } + + hidden [string[]] ParseAllAvailableVersions() { + $url = "https://golang.org/dl/?mode=json&include=all" + $releases = Invoke-RestMethod $url -MaximumRetryCount $this.ApiRetryCount -RetryIntervalSec $this.ApiRetryIntervalSeconds + return $releases.version + } + + hidden [SemVer] FormatVersion([string]$VersionSpec) { + $cleanVersion = $VersionSpec -replace "^go", "" + return [SemVer]$cleanVersion + } + + hidden [bool] ShouldIncludeVersion([SemVer]$Version) { + # For Go, we include all versions greater than 1.12 + return $Version -gt [SemVer]"1.12.0" + } +} \ No newline at end of file diff --git a/get-new-tool-versions/parsers/node-parser.psm1 b/get-new-tool-versions/parsers/node-parser.psm1 new file mode 100644 index 0000000..f15d74f --- /dev/null +++ b/get-new-tool-versions/parsers/node-parser.psm1 @@ -0,0 +1,30 @@ +using module "./base-parser.psm1" + +class NodeVersionsParser: BaseVersionsParser { + [SemVer[]] GetUploadedVersions() { + $url = $this.BuildGitHubFileUrl("actions", "node-versions", "main", "versions-manifest.json") + $releases = Invoke-RestMethod $url -MaximumRetryCount $this.ApiRetryCount -RetryIntervalSec $this.ApiRetryIntervalSeconds + return $releases.version + } + + hidden [string[]] ParseAllAvailableVersions() { + $url = "https://nodejs.org/dist/index.json" + $releases = Invoke-RestMethod $url -MaximumRetryCount $this.ApiRetryCount -RetryIntervalSec $this.ApiRetryIntervalSeconds + return $releases.version + } + + hidden [SemVer] FormatVersion([string]$VersionSpec) { + $cleanVersion = $VersionSpec -replace "^v", "" + return [SemVer]$cleanVersion + } + + hidden [bool] ShouldIncludeVersion([SemVer]$Version) { + if ($Version.Major -lt 8) { + return $false + } + + # For Node.JS, we should include all LTS versions (all even-numbered releases) + # https://nodejs.org/en/about/releases/ + return $Version.Major % 2 -eq 0 + } +} \ No newline at end of file diff --git a/get-new-tool-versions/parsers/parsers-factory.psm1 b/get-new-tool-versions/parsers/parsers-factory.psm1 new file mode 100644 index 0000000..eb9b69c --- /dev/null +++ b/get-new-tool-versions/parsers/parsers-factory.psm1 @@ -0,0 +1,19 @@ +using module "./node-parser.psm1" +using module "./go-parser.psm1" +using module "./python-parser.psm1" + +function Get-ToolVersionsParser { + param( + [Parameter(Mandatory)] + [string]$ToolName + ) + + switch ($ToolName) { + "Node" { return [NodeVersionsParser]::New() } + "Go" { return [GoVersionsParser]::New() } + "Python" { return [PythonVersionsParser]::New() } + Default { + throw "Unknown tool name" + } + } +} \ No newline at end of file diff --git a/get-new-tool-versions/parsers/python-parser.psm1 b/get-new-tool-versions/parsers/python-parser.psm1 new file mode 100644 index 0000000..e82f730 --- /dev/null +++ b/get-new-tool-versions/parsers/python-parser.psm1 @@ -0,0 +1,54 @@ +using module "./base-parser.psm1" + +class PythonVersionsParser: BaseVersionsParser { + [SemVer[]] GetUploadedVersions() { + $url = $this.BuildGitHubFileUrl("actions", "python-versions", "main", "versions-manifest.json") + $releases = Invoke-RestMethod $url -MaximumRetryCount $this.ApiRetryCount -RetryIntervalSec $this.ApiRetryIntervalSeconds + return $releases.version + } + + hidden [string[]] ParseAllAvailableVersions() { + $stableVersionsUrl = "https://www.python.org/ftp/python" + $stableVersionsHtmlRaw = Invoke-WebRequest $stableVersionsUrl + $stableVersionsList = $stableVersionsHtmlRaw.Links.href | Where-Object { + $parsed = $null + return $_.EndsWith("/") -and [SemVer]::TryParse($_.Replace("/", ""), [ref]$parsed) + } + + return $stableVersionsList | ForEach-Object { + $subVersionsUrl = "${stableVersionsUrl}/${_}" + $subVersionsHtmlRaw = Invoke-WebRequest $subVersionsUrl + return $subVersionsHtmlRaw.Links.href | ForEach-Object { + if ($_ -match "^Python-(\d+\.\d+\.\d+[a-z]{0,2}\d*)\.tgz$") { + return $Matches[1] + } + } | ForEach-Object { $_ } | Where-Object { $_ } + } + } + + hidden [SemVer] FormatVersion([string]$VersionSpec) { + $VersionSpec -match "^(\d+)\.(\d+)\.(\d+)([a-z]{1,2})?(\d+)?$" + + if ($Matches.Count -gt 4) { + $VersionLabel = "{0}.{1}" -f $this.ConvertPythonLabel($Matches[4]), $Matches[5] + return [SemVer]::new($Matches[1], $Matches[2], $Matches[3], $VersionLabel) + } + + return [SemVer]::new($Matches[1], $Matches[2], $Matches[3]) + } + + hidden [string] ConvertPythonLabel([string]$Label) { + switch ($Label) { + "a" { return "alpha" } + "b" { return "beta" } + "rc" { return "rc" } + } + + return $Label + } + + [bool] ShouldIncludeVersion([SemVer]$Version) { + # For Go, we include all versions greater than 1.12 + return $Version -gt [SemVer]"3.9.0" + } +} \ No newline at end of file