Merge helpers function from node, boost, python. (#2)

* setup boost helpers

* add ci for pull requests

* pester for ci

* resolving issue with pester 5

* remove Output flag

* check tests with 4 version

* try common tests variant

* added requires

* move to 5 version

* try to add assert module

* fix scope

* use new regex for python

* fix regex in tests

* Pester 4.10.1

* EnableExit

* fix synopsys

* fix creating tar archive

Co-authored-by: Dmitry Shibanov <v-dmshib@microsoft.com>
Co-authored-by: Maxim Lobanov <v-malob@microsoft.com>
pull/4/head
Dmitry Shibanov 5 years ago committed by GitHub
parent 8affb5df76
commit d8c3ce72ee
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -0,0 +1,23 @@
name: Run tests
on: [pull_request]
jobs:
CommonTests:
strategy:
fail-fast: false
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@master
- name: Install Pester
shell: pwsh
run: |
Install-Module Pester -Force -Scope CurrentUser -RequiredVersion 4.10.1
Install-Module Assert -Force -Scope CurrentUser
- name: Run tests
shell: pwsh
run: |
Import-Module Pester
Import-Module Assert
Invoke-Pester -EnableExit

@ -0,0 +1,13 @@
param (
[string] $ToolName
)
$targetPath = $env:AGENT_TOOLSDIRECTORY
if ($ToolName) {
$targetPath = Join-Path $targetPath $ToolName
}
if (Test-Path $targetPath) {
Get-ChildItem -Path $targetPath -Recurse | Where-Object { $_.LinkType -eq "SymbolicLink" } | ForEach-Object { $_.Delete() }
Remove-Item -Path $targetPath -Recurse -Force
}

@ -12,10 +12,15 @@ function Execute-Command {
try { try {
Invoke-Expression $Command | ForEach-Object { Write-Host $_ } Invoke-Expression $Command | ForEach-Object { Write-Host $_ }
if ($LASTEXITCODE -ne 0) { throw "Exit code: $LASTEXITCODE"}
} }
catch { catch {
Write-Host "Error happened during command execution: $Command" $errorMessage = "Error happened during command execution: $Command `n $_"
Write-Host "##vso[task.logissue type=error;] $_" Write-Host $errorMessage
if ($ErrorActionPreference -ne "Continue") {
# avoid logging Azure DevOps issues in case of $ErrorActionPreference -eq Continue
Write-Host "##vso[task.logissue type=error;] $errorMessage"
}
} }
} }
@ -76,5 +81,5 @@ function IsNixPlatform {
[String]$Platform [String]$Platform
) )
return ($Platform -match "macos") -or ($Platform -match "ubuntu") return ($Platform -match "macos") -or ($Platform -match "ubuntu") -or ($Platform -match "linux")
} }

@ -60,9 +60,26 @@ class GitHubApi
return $this.InvokeRestMethod($url, 'Post', $null, $requestBody) return $this.InvokeRestMethod($url, 'Post', $null, $requestBody)
} }
[object] GetGitHubReleases(){ [array] GetReleases(){
$url = "releases" $url = "releases"
return $this.InvokeRestMethod($url, 'GET', $null, $null) $releases = @()
$pageNumber = 1
$releaseNumberLimit = 10000
while ($releases.Count -le $releaseNumberLimit)
{
$requestParams = "page=${pageNumber}&per_page=100"
[array] $response = $this.InvokeRestMethod($url, 'GET', $requestParams, $null)
if ($response.Count -eq 0) {
break
} else {
$releases += $response
$pageNumber++
}
}
return $releases
} }
[string] hidden BuildUrl([string]$Url, [string]$RequestParams) { [string] hidden BuildUrl([string]$Url, [string]$RequestParams) {

@ -0,0 +1,41 @@
<#
.SYNOPSIS
Unpack *.tar file
#>
function Extract-TarArchive {
param(
[Parameter(Mandatory=$true)]
[String]$ArchivePath,
[Parameter(Mandatory=$true)]
[String]$OutputDirectory
)
Write-Debug "Extract $ArchivePath to $OutputDirectory"
tar -C $OutputDirectory -xzf $ArchivePath --strip 1
}
function Create-TarArchive {
param(
[Parameter(Mandatory=$true)]
[String]$SourceFolder,
[Parameter(Mandatory=$true)]
[String]$ArchivePath,
[string]$CompressionType = "gz",
[switch]$DereferenceSymlinks
)
$arguments = @(
"-c", "--$CompressionType"
)
if ($DereferenceSymlinks) {
$arguments += "-h"
}
$arguments += @("-f", $ArchivePath, ".")
Push-Location $SourceFolder
Write-Debug "tar $arguments"
tar @arguments
Pop-Location
}

@ -1,145 +0,0 @@
<#
.SYNOPSIS
Generate versions manifest based on repository releases
.DESCRIPTION
Versions manifest is needed to find the latest assets for particular version of tool
.PARAMETER GitHubRepositoryOwner
Required parameter. The organization which tool repository belongs
.PARAMETER GitHubRepositoryName
Optional parameter. The name of tool repository
.PARAMETER GitHubAccessToken
Required parameter. PAT Token to overcome GitHub API Rate limit
.PARAMETER OutputFile
Required parameter. File "*.json" where generated results will be saved
.PARAMETER PlatformMapFile
Optional parameter. Path to the json file with platform map
Structure example:
{
"macos-1014": [
{
"platform": "darwin",
"platform_version": "10.14"
}, ...
], ...
}
#>
param (
[Parameter(Mandatory)] [string] $GitHubRepositoryOwner,
[Parameter(Mandatory)] [string] $GitHubRepositoryName,
[Parameter(Mandatory)] [string] $GitHubAccessToken,
[Parameter(Mandatory)] [string] $OutputFile,
[string] $PlatformMapFile
)
Import-Module (Join-Path $PSScriptRoot "../github/github-api.psm1")
if ($PlatformMapFile -and (Test-Path $PlatformMapFile)) {
$PlatformMap = Get-Content $PlatformMapFile -Raw | ConvertFrom-Json -AsHashtable
} else {
$PlatformMap = @{}
}
function New-AssetItem {
param (
[Parameter(Mandatory)][string]$Filename,
[Parameter(Mandatory)][string]$DownloadUrl,
[Parameter(Mandatory)][string]$Arch,
[Parameter(Mandatory)][string]$Platform,
[string]$PlatformVersion
)
$asset = New-Object PSObject
$asset | Add-Member -Name "filename" -Value $Filename -MemberType NoteProperty
$asset | Add-Member -Name "arch" -Value $Arch -MemberType NoteProperty
$asset | Add-Member -Name "platform" -Value $Platform -MemberType NoteProperty
if ($PlatformVersion) { $asset | Add-Member -Name "platform_version" -Value $PlatformVersion -MemberType NoteProperty }
$asset | Add-Member -Name "download_url" -Value $DownloadUrl -MemberType NoteProperty
return $asset
}
function Build-AssetsList {
param (
[AllowEmptyCollection()]
[Parameter(Mandatory)][array]$ReleaseAssets
)
$assets = @()
foreach($releaseAsset in $ReleaseAssets) {
$parts = [IO.path]::GetFileNameWithoutExtension($releaseAsset.name).Split("-")
$arch = $parts[-1]
$buildPlatform = [string]::Join("-", $parts[2..($parts.Length-2)])
if ($PlatformMap[$buildPlatform]) {
$PlatformMap[$buildPlatform] | ForEach-Object {
$assets += New-AssetItem -Filename $releaseAsset.name `
-DownloadUrl $releaseAsset.browser_download_url `
-Arch $arch `
-Platform $_.platform `
-PlatformVersion $_.platform_version
}
} else {
$assets += New-AssetItem -Filename $releaseAsset.name `
-DownloadUrl $releaseAsset.browser_download_url `
-Arch $arch `
-Platform $buildPlatform
}
}
return $assets
}
function Get-VersionFromRelease {
param (
[Parameter(Mandatory)][object]$Release
)
# Release name can contain additional information after ':' so filter it
[string]$releaseName = $Release.name.Split(':')[0]
[Version]$version = $null
if (![Version]::TryParse($releaseName, [ref]$version)) {
throw "Release '$($Release.id)' has invalid title '$($Release.name)'. It can't be parsed as version. ( $($Release.html_url) )"
}
return $version
}
function Build-VersionsManifest {
param (
[Parameter(Mandatory)][array]$Releases
)
$Releases = $Releases | Sort-Object -Property "published_at" -Descending
$versionsHash = @{}
foreach ($release in $Releases) {
if (($release.draft -eq $true) -or ($release.prerelease -eq $true)) {
continue
}
[Version]$version = Get-VersionFromRelease $release
$versionKey = $version.ToString()
if ($versionsHash.ContainsKey($versionKey)) {
continue
}
$versionsHash.Add($versionKey, [PSCustomObject]@{
version = $versionKey
stable = $true
release_url = $release.html_url
files = Build-AssetsList $release.assets
})
}
# Sort versions by descending
return $versionsHash.Values | Sort-Object -Property @{ Expression = { [Version]$_.version }; Descending = $true }
}
$gitHubApi = Get-GitHubApi -AccountName $GitHubRepositoryOwner -ProjectName $GitHubRepositoryName -AccessToken $GitHubAccessToken
$releases = $gitHubApi.GetGitHubReleases()
$versionIndex = Build-VersionsManifest $releases
$versionIndex | ConvertTo-Json -Depth 5 | Out-File $OutputFile -Encoding utf8 -Force

@ -0,0 +1,35 @@
<#
.SYNOPSIS
Generate versions manifest based on repository releases
.DESCRIPTION
Versions manifest is needed to find the latest assets for particular version of tool
.PARAMETER GitHubRepositoryOwner
Required parameter. The organization which tool repository belongs
.PARAMETER GitHubRepositoryName
Required parameter. The name of tool repository
.PARAMETER GitHubAccessToken
Required parameter. PAT Token to overcome GitHub API Rate limit
.PARAMETER OutputFile
Required parameter. File "*.json" where generated results will be saved
.PARAMETER ConfigurationFile
Path to the json file with parsing configuration
#>
param (
[Parameter(Mandatory)] [string] $GitHubRepositoryOwner,
[Parameter(Mandatory)] [string] $GitHubRepositoryName,
[Parameter(Mandatory)] [string] $GitHubAccessToken,
[Parameter(Mandatory)] [string] $OutputFile,
[Parameter(Mandatory)] [string] $ConfigurationFile
)
Import-Module (Join-Path $PSScriptRoot "../github/github-api.psm1")
Import-Module (Join-Path $PSScriptRoot "manifest-utils.psm1") -Force
$configuration = Read-ConfigurationFile -Filepath $ConfigurationFile
$gitHubApi = Get-GitHubApi -AccountName $GitHubRepositoryOwner -ProjectName $GitHubRepositoryName -AccessToken $GitHubAccessToken
$releases = $gitHubApi.GetReleases()
$versionIndex = Build-VersionsManifest -Releases $releases -Configuration $configuration
$versionIndex | ConvertTo-Json -Depth 5 | Out-File $OutputFile -Encoding UTF8NoBOM -Force

@ -0,0 +1,119 @@
#Requires -Modules Pester
#Requires -Modules Assert
Import-Module (Join-Path $PSScriptRoot "manifest-utils.psm1") -Force
Describe "New-AssetItem" {
It "use regex to parse all values in correct order" {
$githubAsset = @{ name = "python-3.8.3-linux-16.04-x64.tar.gz"; browser_download_url = "long_url"; }
$configuration = @{
regex = "python-\d+\.\d+\.\d+-(\w+)-([\w\.]+)?-?(x\d+)";
groups = [PSCustomObject]@{ platform = 1; platform_version = 2; arch = 3; };
}
$expectedOutput = [PSCustomObject]@{
filename = "python-3.8.3-linux-16.04-x64.tar.gz"; platform = "linux"; platform_version = "16.04";
arch = "x64"; download_url = "long_url";
}
$actualOutput = New-AssetItem -ReleaseAsset $githubAsset -Configuration $configuration
Assert-Equivalent -Actual $actualOutput -Expected $expectedOutput
}
It "support constant values in groups" {
$githubAsset = @{ name = "python-3.8.3-linux-16.04-x64.tar.gz"; browser_download_url = "long_url"; }
$configuration = @{
regex = "python-\d+\.\d+\.\d+-(\w+)-([\w\.]+)?-?(x\d+)";
groups = [PSCustomObject]@{ platform = 1; platform_version = 2; arch = "x64"; }
}
$expectedOutput = [PSCustomObject]@{
filename = "python-3.8.3-linux-16.04-x64.tar.gz"; platform = "linux"; platform_version = "16.04";
arch = "x64"; download_url = "long_url";
}
$actualOutput = New-AssetItem -ReleaseAsset $githubAsset -Configuration $configuration
Assert-Equivalent -Actual $actualOutput -Expected $expectedOutput
}
It "Skip empty groups" {
$githubAsset = @{ name = "python-3.8.3-win32-x64.zip"; browser_download_url = "long_url"; }
$configuration = @{
regex = "python-\d+\.\d+\.\d+-(\w+)-([\w\.]+)?-?(x\d+)";
groups = [PSCustomObject]@{ platform = 1; platform_version = 2; arch = 3; }
}
$expectedOutput = [PSCustomObject]@{
filename = "python-3.8.3-win32-x64.zip"; platform = "win32";
arch = "x64"; download_url = "long_url";
}
$actualOutput = New-AssetItem -ReleaseAsset $githubAsset -Configuration $configuration
Assert-Equivalent -Actual $actualOutput -Expected $expectedOutput
}
}
Describe "Get-VersionFromRelease" {
It "clear version" {
$release = @{ name = "3.8.3" }
Get-VersionFromRelease -Release $release | Should -Be "3.8.3"
}
It "version with title" {
$release = @{ name = "3.8.3: Release title" }
Get-VersionFromRelease -Release $release | Should -Be "3.8.3"
}
}
Describe "Build-VersionsManifest" {
$assets = @(
@{ name = "python-3.8.3-linux-16.04-x64.tar.gz"; browser_download_url = "fake_url"; }
@{ name = "python-3.8.3-linux-18.04-x64.tar.gz"; browser_download_url = "fake_url"; }
)
$configuration = @{
regex = "python-\d+\.\d+\.\d+-(\w+)-([\w\.]+)?-?(x\d+)";
groups = [PSCustomObject]@{ platform = 1; platform_version = 2; arch = "x64"; }
}
$expectedManifestFiles = @(
[PSCustomObject]@{ filename = "python-3.8.3-linux-16.04-x64.tar.gz"; arch = "x64"; platform = "linux"; platform_version = "16.04"; download_url = "fake_url" },
[PSCustomObject]@{ filename = "python-3.8.3-linux-18.04-x64.tar.gz"; arch = "x64"; platform = "linux"; platform_version = "18.04"; download_url = "fake_url" }
)
It "build manifest with correct version order" {
$releases = @(
@{ name = "3.8.1"; draft = $false; prerelease = $false; html_url = "fake_html_url"; published_at = "2020-05-14T09:54:06Z"; assets = $assets },
@{ name = "3.5.2: Hello"; draft = $false; prerelease = $false; html_url = "fake_html_url"; published_at = "2020-05-06T11:45:36Z"; assets = $assets },
@{ name = "3.8.3: Release title"; draft = $false; prerelease = $false; html_url = "fake_html_url"; published_at = "2020-05-06T11:43:38Z"; assets = $assets }
)
$expectedManifest = @(
[PSCustomObject]@{ version = "3.8.3"; stable = $true; release_url = "fake_html_url"; files = $expectedManifestFiles },
[PSCustomObject]@{ version = "3.8.1"; stable = $true; release_url = "fake_html_url"; files = $expectedManifestFiles },
[PSCustomObject]@{ version = "3.5.2"; stable = $true; release_url = "fake_html_url"; files = $expectedManifestFiles }
)
$actualManifest = Build-VersionsManifest -Releases $releases -Configuration $configuration
Assert-Equivalent -Actual $actualManifest -Expected $expectedManifest
}
It "Skip draft and prerelease" {
$releases = @(
@{ name = "3.8.1"; draft = $true; prerelease = $false; html_url = "fake_html_url"; published_at = "2020-05-14T09:54:06Z"; assets = $assets },
@{ name = "3.5.2"; draft = $false; prerelease = $true; html_url = "fake_html_url"; published_at = "2020-05-06T11:45:36Z"; assets = $assets },
@{ name = "3.8.3"; draft = $false; prerelease = $false; html_url = "fake_html_url"; published_at = "2020-05-06T11:43:38Z"; assets = $assets }
)
$expectedManifest = @(
[PSCustomObject]@{ version = "3.8.3"; stable = $true; release_url = "fake_html_url"; files = $expectedManifestFiles }
)
[array]$actualManifest = Build-VersionsManifest -Releases $releases -Configuration $configuration
Assert-Equivalent -Actual $actualManifest -Expected $expectedManifest
}
It "take latest published release for each version" {
$releases = @(
@{ name = "3.8.1"; draft = $false; prerelease = $false; html_url = "fake_html_url1"; published_at = "2020-05-06T11:45:36Z"; assets = $assets },
@{ name = "3.8.1"; draft = $false; prerelease = $false; html_url = "fake_html_url2"; published_at = "2020-05-14T09:54:06Z"; assets = $assets },
@{ name = "3.8.1"; draft = $false; prerelease = $false; html_url = "fake_html_url3"; published_at = "2020-05-06T11:43:38Z"; assets = $assets }
)
$expectedManifest = @(
[PSCustomObject]@{ version = "3.8.1"; stable = $true; release_url = "fake_html_url2"; files = $expectedManifestFiles }
)
[array]$actualManifest = Build-VersionsManifest -Releases $releases -Configuration $configuration
Assert-Equivalent -Actual $actualManifest -Expected $expectedManifest
}
}

@ -0,0 +1,77 @@
function Read-ConfigurationFile {
param ([Parameter(Mandatory)][string]$Filepath)
return Get-Content $Filepath -Raw | ConvertFrom-Json
}
function New-AssetItem {
param (
[Parameter(Mandatory)][object]$ReleaseAsset,
[Parameter(Mandatory)][object]$Configuration
)
$regexResult = [regex]::Match($ReleaseAsset.name, $Configuration.regex)
if (-not $regexResult.Success) { throw "Can't match asset filename '$($_.name)' to regex" }
$result = New-Object PSObject
$result | Add-Member -Name "filename" -Value $ReleaseAsset.name -MemberType NoteProperty
$Configuration.groups.PSObject.Properties | ForEach-Object {
if (($_.Value).GetType().Name.StartsWith("Int")) {
$value = $regexResult.Groups[$_.Value].Value
} else {
$value = $_.Value
}
if (-not ([string]::IsNullOrEmpty($value))) {
$result | Add-Member -Name $_.Name -Value $value -MemberType NoteProperty
}
}
$result | Add-Member -Name "download_url" -Value $ReleaseAsset.browser_download_url -MemberType NoteProperty
return $result
}
function Get-VersionFromRelease {
param (
[Parameter(Mandatory)][object]$Release
)
# Release name can contain additional information after ':' so filter it
[string]$releaseName = $Release.name.Split(':')[0]
[Version]$version = $null
if (![Version]::TryParse($releaseName, [ref]$version)) {
throw "Release '$($Release.id)' has invalid title '$($Release.name)'. It can't be parsed as version. ( $($Release.html_url) )"
}
return $version
}
function Build-VersionsManifest {
param (
[Parameter(Mandatory)][array]$Releases,
[Parameter(Mandatory)][object]$Configuration
)
$Releases = $Releases | Sort-Object -Property "published_at" -Descending
$versionsHash = @{}
foreach ($release in $Releases) {
if (($release.draft -eq $true) -or ($release.prerelease -eq $true)) {
continue
}
[Version]$version = Get-VersionFromRelease $release
$versionKey = $version.ToString()
if ($versionsHash.ContainsKey($versionKey)) {
continue
}
$versionsHash.Add($versionKey, [PSCustomObject]@{
version = $versionKey
stable = $true
release_url = $release.html_url
files = $release.assets | ForEach-Object { New-AssetItem -ReleaseAsset $_ -Configuration $Configuration }
})
}
# Sort versions by descending
return $versionsHash.Values | Sort-Object -Property @{ Expression = { [Version]$_.version }; Descending = $true }
}

@ -1,34 +0,0 @@
<#
.SYNOPSIS
Pack folder to *.zip format
#>
function Pack-Zip {
param(
[Parameter(Mandatory=$true)]
[String]$PathToArchive,
[Parameter(Mandatory=$true)]
[String]$ToolZipFile
)
Write-Debug "Pack $PathToArchive to $ToolZipFile"
Push-Location -Path $PathToArchive
zip -q -r $ToolZipFile * | Out-Null
Pop-Location
}
<#
.SYNOPSIS
Unpack *.tar file
#>
function Unpack-TarArchive {
param(
[Parameter(Mandatory=$true)]
[String]$ArchivePath,
[Parameter(Mandatory=$true)]
[String]$OutputDirectory
)
Write-Debug "Unpack $ArchivePath to $OutputDirectory"
tar -C $OutputDirectory -xzf $ArchivePath
}

@ -0,0 +1,39 @@
<#
.SYNOPSIS
Unpack *.7z file
#>
function Extract-SevenZipArchive {
param(
[Parameter(Mandatory=$true)]
[String]$ArchivePath,
[Parameter(Mandatory=$true)]
[String]$OutputDirectory
)
Write-Debug "Extract $ArchivePath to $OutputDirectory"
7z x $ArchivePath -o"$OutputDirectory" -y | Out-Null
}
function Create-SevenZipArchive {
param(
[Parameter(Mandatory=$true)]
[String]$SourceFolder,
[Parameter(Mandatory=$true)]
[String]$ArchivePath,
[String]$ArchiveType = "zip",
[String]$CompressionLevel = 5,
[switch]$IncludeSymlinks
)
$ArchiveTypeArguments = @(
"-t${ArchiveType}",
"-mx=${CompressionLevel}"
)
if ($IncludeSymlinks) {
$ArchiveTypeArguments += "-snl"
}
Push-Location $SourceFolder
Write-Debug "7z a $ArchiveTypeArgument $ArchivePath @$SourceFolder"
7z a @ArchiveTypeArguments $ArchivePath $SourceFolder\*
Pop-Location
}

@ -0,0 +1,48 @@
###
# Visual Studio helper functions
###
function Get-VSWhere {
$vswhere = "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe";
if (-not (Test-Path $vswhere )) {
[Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12
$vswhere = ".\vswhere.exe"
$vswhereApiUri = "https://api.github.com/repos/Microsoft/vswhere/releases/latest"
$tag = (Invoke-RestMethod -Uri $vswhereApiUri)[0].tag_name
$vswhereUri = "https://github.com/Microsoft/vswhere/releases/download/$tag/vswhere.exe"
Invoke-WebRequest -Uri $vswhereUri -OutFile $vswhere | Out-Null
}
return $vswhere
}
function Invoke-Environment
{
Param
(
[Parameter(Mandatory)]
[string]
$Command
)
& "${env:COMSPEC}" /s /c "`"$Command`" -no_logo && set" | Foreach-Object {
if ($_ -match '^([^=]+)=(.*)') {
[System.Environment]::SetEnvironmentVariable($matches[1], $matches[2])
}
}
}
function Get-VSInstallationPath {
$vswhere = Get-VSWhere
$installationPath = & $vswhere -prerelease -legacy -latest -property installationPath
return $installationPath
}
function Invoke-VSDevEnvironment {
Write-Host "Invoke-VSDevEnvironment had been invoked"
$installationPath = Get-VSInstallationPath
$envFilepath = Join-Path $installationPath "Common7\Tools\vsdevcmd.bat"
Invoke-Environment -Command $envFilepath
}
Loading…
Cancel
Save