From c455148af9c78ce268d0362b4701fb7477f38a05 Mon Sep 17 00:00:00 2001 From: Nikita Bykov <49442273+nikita-bykov@users.noreply.github.com> Date: Wed, 26 Aug 2020 13:18:15 +0300 Subject: [PATCH 1/2] Migrate go-versions CI to GitHub Actions (#12) * migrated Go to GA * commented tests * commented tests * fixed upload * fixed trigger_pr * fixed trigger_pr * fixed tests * removed comment * updated build-go-packages and create-pr * update * updated helpers * fixed build-go-packages * updated build-go-packages.yml * updated build-go.ps1 * updated Go.Tests.ps1 * fixed tests * fixed go tests * fixed go tests * fixed tests * fixed "version is correct" test * updated "version is correct" test * fixed Invoke-Expression * Remove Create-ArtifactDirectories function * fixed tests * fixed tests * fix test * fixed trigger_pr * Update to Pester 5 * update * updated * fixed test * fixed tests * fixed tests Co-authored-by: Nikita Bykov --- .github/workflows/build-go-packages.yml | 167 ++++++++++++++++++++++++ .github/workflows/create-pr.yml | 33 +++++ CONTRIBUTING.md | 10 +- azure-pipelines/build-go-packages.yml | 65 --------- azure-pipelines/templates/build-job.yml | 21 --- azure-pipelines/templates/test-job.yml | 75 ----------- builders/go-builder.psm1 | 18 ++- helpers | 2 +- tests/Go.Tests.ps1 | 36 ++--- 9 files changed, 237 insertions(+), 190 deletions(-) create mode 100644 .github/workflows/build-go-packages.yml create mode 100644 .github/workflows/create-pr.yml delete mode 100644 azure-pipelines/build-go-packages.yml delete mode 100644 azure-pipelines/templates/build-job.yml delete mode 100644 azure-pipelines/templates/test-job.yml diff --git a/.github/workflows/build-go-packages.yml b/.github/workflows/build-go-packages.yml new file mode 100644 index 0000000..8f9b676 --- /dev/null +++ b/.github/workflows/build-go-packages.yml @@ -0,0 +1,167 @@ +name: Generate Go package +on: + workflow_dispatch: + inputs: + VERSION: + description: 'Go version to build and upload' + required: true + default: '1.15.0' + PUBLISH_RELEASES: + description: 'Whether to publish releases' + required: true + default: 'false' + +env: + VERSION: ${{ github.event.inputs.VERSION }} +defaults: + run: + shell: pwsh + +jobs: + build_go: + name: Build Go ${{ github.event.inputs.VERSION }} [${{ matrix.platform }}] + runs-on: ubuntu-latest + env: + ARTIFACT_NAME: go-${{ github.event.inputs.VERSION }}-${{ matrix.platform }}-x64 + strategy: + fail-fast: false + matrix: + platform: [linux, darwin, win32] + steps: + - uses: actions/checkout@v2 + with: + submodules: true + + - name: Build Go ${{ env.VERSION }} + shell: pwsh + run: | + ./builders/build-go.ps1 -Version $env:VERSION ` + -Platform ${{ matrix.platform }} + + - name: Publish artifact + uses: actions/upload-artifact@v2 + with: + name: ${{ env.ARTIFACT_NAME }} + path: ${{ runner.temp }}/artifact + + test_go: + name: Test Go ${{ github.event.inputs.VERSION }} [${{ matrix.platform }}] + needs: build_go + runs-on: ${{ matrix.os }} + env: + ARTIFACT_NAME: go-${{ github.event.inputs.VERSION }}-${{ matrix.platform }}-x64 + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-latest + platform: linux + - os: macos-latest + platform: darwin + - os: windows-latest + platform: win32 + steps: + - uses: actions/checkout@v2 + with: + submodules: true + + - name: Fully cleanup the toolcache directory before testing + run: ./helpers/clean-toolcache.ps1 -ToolName "go" + + - name: Download artifact + uses: actions/download-artifact@v2 + with: + path: ${{ runner.temp }} + + - name: Extract files + run: | + if ('${{ matrix.platform }}' -eq 'win32') { + $artifactName = "${{ env.ARTIFACT_NAME }}.zip" + 7z.exe x "$artifactName" -y | Out-Null + } else { + $artifactName = "${{ env.ARTIFACT_NAME }}.tar.gz" + tar -xzf $artifactName + } + working-directory: ${{ runner.temp }}/${{ env.ARTIFACT_NAME }} + + - name: Apply build artifact to the local machine + run: | + if ('${{ matrix.platform }}' -eq 'win32') { powershell ./setup.ps1 } else { sh ./setup.sh } + working-directory: ${{ runner.temp }}/${{ env.ARTIFACT_NAME }} + + - name: Setup Go ${{ env.VERSION }} + uses: actions/setup-go@v2.1.1 + with: + go-version: ${{ env.VERSION }} + + - name: Wait for the logs + run: | + Write-Host "Fake step that do nothing" + Write-Host "We need it because log of previous step 'Setup Go' is not available here yet." + Write-Host "In testing step (Go.Tests.ps1) we analyze build log of 'Setup Go' task" + Write-Host "to determine if Go.js version was consumed from cache and was downloaded" + Write-Host "Random values:" + for ($i = 0; $i -lt 200; $i++) { Get-Random } + + - name: Run tests + run: | + Install-Module Pester -Force -Scope CurrentUser + Import-Module Pester + Invoke-Pester -Script ./Go.Tests.ps1 -EnableExit + working-directory: ./tests + + publish_release: + name: Publish release + if: github.event.inputs.PUBLISH_RELEASES == 'true' + needs: test_go + runs-on: ubuntu-latest + outputs: + upload_url: ${{ steps.create_release.outputs.upload_url }} + steps: + - uses: actions/download-artifact@v2 + + - name: Publish Release ${{ env.VERSION }} + id: create_release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: ${{ env.VERSION }}-${{ github.run_id }} + release_name: ${{ env.VERSION }} + body: | + Upload Go ${{ env.VERSION }} + + - name: Upload release assets + uses: actions/github-script@v2 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const fs = require('fs'); + for (let artifactDir of fs.readdirSync('.')) { + let artifactName = fs.readdirSync(`${artifactDir}`)[0]; + console.log(`Upload ${artifactName} asset`); + github.repos.uploadReleaseAsset({ + owner: context.repo.owner, + repo: context.repo.repo, + release_id: ${{ steps.create_release.outputs.id }}, + name: artifactName, + data: fs.readFileSync(`./${artifactDir}/${artifactName}`) + }); + } + + trigger_pr: + name: Trigger "Create Pull Request" workflow + needs: publish_release + runs-on: ubuntu-latest + steps: + - name: Trigger "Create Pull Request" workflow + uses: actions/github-script@v2 + with: + github-token: ${{ secrets.PERSONAL_TOKEN }} + script: | + github.actions.createWorkflowDispatch({ + owner: context.repo.owner, + repo: context.repo.repo, + workflow_id: 'create-pr.yml', + ref: 'main' + }); diff --git a/.github/workflows/create-pr.yml b/.github/workflows/create-pr.yml new file mode 100644 index 0000000..020e600 --- /dev/null +++ b/.github/workflows/create-pr.yml @@ -0,0 +1,33 @@ +name: Create Pull Request +on: + workflow_dispatch: + +defaults: + run: + shell: pwsh + +jobs: + create_pr: + name: Create Pull Request + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + submodules: true + + - name: Create versions-manifest.json + run: | + ./helpers/packages-generation/manifest-generator.ps1 -RepositoryFullName "$env:GITHUB_REPOSITORY" ` + -GitHubAccessToken "${{secrets.GITHUB_TOKEN}}" ` + -OutputFile "./versions-manifest.json" ` + -ConfigurationFile "./config/go-manifest-config.json" + - name: Create GitHub PR + run: | + $formattedDate = Get-Date -Format "MM/dd/yyyy" + ./helpers/github/create-pull-request.ps1 ` + -RepositoryFullName "$env:GITHUB_REPOSITORY" ` + -AccessToken "${{secrets.GITHUB_TOKEN}}" ` + -BranchName "update-versions-manifest-file" ` + -CommitMessage "Update versions-manifest" ` + -PullRequestTitle "[versions-manifest] Update for release from ${formattedDate}" ` + -PullRequestBody "Update versions-manifest.json for release from ${formattedDate}" \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5b71128..a0047c1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -29,22 +29,20 @@ Here are a few things you can do that will increase the likelihood of your pull ### Directory structure ``` -├── azure-pipelines/ -| └──templates/ +├── .github/ +| └──workflows/ ├── builders/ ├── helpers/ ├── installers/ └── tests/ └──sources/ ``` -- `azure-pipelines*` - contains global YAML definitions for build pipelines. Reusable templates for specific jobs are located in `templates` subfolder. +- `.github/workflows` - contains repository workflow files. - `builders` - contains Go builder classes and functions. -- `helpers` - contains global helper functions and functions. +- `helpers` - contains global helper classes and functions. - `installers` - contains installation script templates. - `tests` - contains test scripts. Required tests sources are located in `sources` subfolder. -\* _We use Azure Pipelines because there are a few features that Actions is still missing, we'll move to Actions as soon as possible_. - ## Resources - [How to Contribute to Open Source](https://opensource.guide/how-to-contribute/) diff --git a/azure-pipelines/build-go-packages.yml b/azure-pipelines/build-go-packages.yml deleted file mode 100644 index 148b81e..0000000 --- a/azure-pipelines/build-go-packages.yml +++ /dev/null @@ -1,65 +0,0 @@ -name: $(date:yyyyMMdd)$(rev:.r)-Go-$(VERSION) -trigger: none -pr: - autoCancel: true - branches: - include: - - main - paths: - exclude: - - versions-manifest.json - -stages: -- stage: Build_Go_Darwin - dependsOn: [] - variables: - Platform: darwin - Architecture: x64 - jobs: - - template: /azure-pipelines/templates/build-job.yml - -- stage: Test_Go_Darwin - condition: succeeded() - dependsOn: Build_Go_Darwin - variables: - VmImage: macOS-latest - Platform: darwin - Architecture: x64 - jobs: - - template: /azure-pipelines/templates/test-job.yml - -- stage: Build_Go_Linux - dependsOn: [] - variables: - Platform: linux - Architecture: x64 - jobs: - - template: /azure-pipelines/templates/build-job.yml - -- stage: Test_Go_Linux - condition: succeeded() - dependsOn: Build_Go_Linux - variables: - VmImage: ubuntu-latest - Platform: linux - Architecture: x64 - jobs: - - template: /azure-pipelines/templates/test-job.yml - -- stage: Build_Go_Windows - dependsOn: [] - variables: - Platform: win32 - Architecture: x64 - jobs: - - template: /azure-pipelines/templates/build-job.yml - -- stage: Test_Go_Windows - condition: succeeded() - dependsOn: Build_Go_Windows - variables: - VmImage: windows-latest - Platform: win32 - Architecture: x64 - jobs: - - template: /azure-pipelines/templates/test-job.yml \ No newline at end of file diff --git a/azure-pipelines/templates/build-job.yml b/azure-pipelines/templates/build-job.yml deleted file mode 100644 index 2d8579b..0000000 --- a/azure-pipelines/templates/build-job.yml +++ /dev/null @@ -1,21 +0,0 @@ -jobs: -- job: Build_Go - timeoutInMinutes: 90 - pool: - name: Azure Pipelines - vmImage: ubuntu-latest - steps: - - checkout: self - - - task: PowerShell@2 - displayName: 'Build Go $(Version)' - inputs: - targetType: filePath - filePath: './builders/build-go.ps1' - arguments: '-Version $(Version) -Platform $(Platform) -Architecture $(Architecture)' - - - task: PublishPipelineArtifact@1 - displayName: 'Publish Artifact: Go $(Version)' - inputs: - targetPath: '$(Build.ArtifactStagingDirectory)' - artifactName: 'go-$(Version)-$(Platform)-$(Architecture)' \ No newline at end of file diff --git a/azure-pipelines/templates/test-job.yml b/azure-pipelines/templates/test-job.yml deleted file mode 100644 index 9174712..0000000 --- a/azure-pipelines/templates/test-job.yml +++ /dev/null @@ -1,75 +0,0 @@ -jobs: -- job: Test_Go - pool: - name: Azure Pipelines - vmImage: $(VmImage) - steps: - - checkout: self - submodules: true - - - task: PowerShell@2 - displayName: Fully cleanup the toolcache directory before testing - inputs: - targetType: filePath - filePath: helpers/clean-toolcache.ps1 - arguments: -ToolName "go" - - - task: DownloadPipelineArtifact@2 - inputs: - source: 'current' - artifact: 'go-$(Version)-$(Platform)-$(Architecture)' - path: $(Build.ArtifactStagingDirectory) - - - task: ExtractFiles@1 - inputs: - archiveFilePatterns: '$(Build.ArtifactStagingDirectory)/go-$(Version)-$(Platform)-$(Architecture).*' - destinationFolder: $(Build.BinariesDirectory) - cleanDestinationFolder: false - - - task: PowerShell@2 - displayName: 'Apply build artifact to the local machines' - inputs: - targetType: inline - script: | - if ("$(Platform)" -match 'win32') { powershell ./setup.ps1 } else { sh ./setup.sh } - workingDirectory: '$(Build.BinariesDirectory)' - - - task: GoTool@0 - inputs: - version: '$(Version)' - - - task: PowerShell@2 - displayName: 'Wait for the logs' - inputs: - targetType: inline - script: | - Write-Host "Fake step that do nothing" - Write-Host "We need it because log of previous step 'Use Go' is not available here yet." - Write-Host "In testing step (Go.Tests.ps1) we analyze build log of 'GoTool' task" - Write-Host "to determine if Go version was consumed from cache and was downloaded" - - - task: PowerShell@2 - displayName: 'Run tests' - inputs: - TargetType: inline - script: | - Install-Module Pester -Force -Scope CurrentUser -RequiredVersion 4.10.1 - Import-Module Pester - $pesterParams = @{ - Path="./Go.Tests.ps1"; - Parameters=@{ - Version="$(Version)"; - } - } - Invoke-Pester -Script $pesterParams -OutputFile "$(Build.SourcesDirectory)/tests/test_results.xml" -OutputFormat NUnitXml - workingDirectory: '$(Build.SourcesDirectory)/tests' - - - task: PublishTestResults@2 - displayName: 'Publish test results' - inputs: - testResultsFiles: '*.xml' - testResultsFormat: NUnit - searchFolder: 'tests' - failTaskOnFailedTests: true - testRunTitle: "Go $(Version)-$(Platform)" - condition: always() diff --git a/builders/go-builder.psm1 b/builders/go-builder.psm1 index e65cef8..0181040 100644 --- a/builders/go-builder.psm1 +++ b/builders/go-builder.psm1 @@ -16,10 +16,13 @@ class GoBuilder { The architecture with which Go should be built. .PARAMETER TempFolderLocation - The location of temporary files that will be used during Go package generation. Using system BUILD_STAGINGDIRECTORY variable value. + The location of temporary files that will be used during Go package generation. - .PARAMETER ArtifactLocation - The location of generated Go artifact. Using system environment BUILD_BINARIESDIRECTORY variable value. + .PARAMETER WorkFolderLocation + The location of installation files. + + .PARAMETER ArtifactFolderLocation + The location of generated Go artifact. .PARAMETER InstallationTemplatesLocation The location of installation script template. Using "installers" folder from current repository. @@ -40,9 +43,8 @@ class GoBuilder { $this.Architecture = $architecture $this.TempFolderLocation = [IO.Path]::GetTempPath() - $this.WorkFolderLocation = $env:BUILD_BINARIESDIRECTORY - $this.ArtifactFolderLocation = $env:BUILD_STAGINGDIRECTORY - + $this.WorkFolderLocation = Join-Path $env:RUNNER_TEMP "binaries" + $this.ArtifactFolderLocation = Join-Path $env:RUNNER_TEMP "artifact" $this.InstallationTemplatesLocation = Join-Path -Path $PSScriptRoot -ChildPath "../installers" } @@ -95,6 +97,10 @@ class GoBuilder { Generates Go artifact from downloaded binaries. #> + Write-Host "Create WorkFolderLocation and ArtifactFolderLocation folders" + New-Item -Path $this.WorkFolderLocation -ItemType "directory" + New-Item -Path $this.ArtifactFolderLocation -ItemType "directory" + Write-Host "Download Go $($this.Version) [$($this.Architecture)] executable..." $binariesArchivePath = $this.Download() diff --git a/helpers b/helpers index 68072be..3b38e3d 160000 --- a/helpers +++ b/helpers @@ -1 +1 @@ -Subproject commit 68072bedefb41436c6b70ddfa9adb8e631a3b6cf +Subproject commit 3b38e3de4c5e4bc75f5dee12b5bb8dbffe35c562 diff --git a/tests/Go.Tests.ps1 b/tests/Go.Tests.ps1 index 705e7f1..598cc02 100644 --- a/tests/Go.Tests.ps1 +++ b/tests/Go.Tests.ps1 @@ -1,27 +1,30 @@ -param ( - [version] [Parameter (Mandatory = $true)] [ValidateNotNullOrEmpty()] - $Version -) - Import-Module (Join-Path $PSScriptRoot "../helpers/pester-extensions.psm1") Import-Module (Join-Path $PSScriptRoot "../helpers/common-helpers.psm1") -function Get-UseGoLogs { - $logsFolderPath = Join-Path -Path $env:AGENT_HOMEDIRECTORY -ChildPath "_diag" | Join-Path -ChildPath "pages" +BeforeAll { + Set-Location -Path "source" + $sourceLocation = Get-Location + function Get-UseGoLogs { + # GitHub Windows images don't have `HOME` variable + $homeDir = $env:HOME ?? $env:HOMEDRIVE + $logsFolderPath = Join-Path -Path $homeDir -ChildPath "runners/*/_diag/pages" -Resolve - $useGoLogFile = Get-ChildItem -Path $logsFolderPath | Where-Object { - $logContent = Get-Content $_.Fullname -Raw - return $logContent -match "GoTool" - } | Select-Object -First 1 - return $useGoLogFile.Fullname + $useGoLogFile = Get-ChildItem -Path $logsFolderPath | Where-Object { + $logContent = Get-Content $_.Fullname -Raw + return $logContent -match "setup-go@v" + } | Select-Object -First 1 + return $useGoLogFile.Fullname + } } + Describe "Go" { It "is available" { "go version" | Should -ReturnZeroExitCode } It "version is correct" { + [version]$Version = $env:VERSION $versionOutput = Invoke-Expression -Command "go version" $finalVersion = $Version.ToString(3) If ($Version.Build -eq "0"){ @@ -33,7 +36,10 @@ Describe "Go" { It "is used from tool-cache" { $goPath = (Get-Command "go").Path $goPath | Should -Not -BeNullOrEmpty - $expectedPath = Join-Path -Path $env:AGENT_TOOLSDIRECTORY -ChildPath "go" + + # GitHub Windows images don't have `AGENT_TOOLSDIRECTORY` variable + $toolcacheDir = $env:AGENT_TOOLSDIRECTORY ?? $env:RUNNER_TOOL_CACHE + $expectedPath = Join-Path -Path $toolcacheDir -ChildPath "go" $goPath.startsWith($expectedPath) | Should -BeTrue -Because "'$goPath' is not started with '$expectedPath'" } @@ -42,11 +48,9 @@ Describe "Go" { $useGoLogFile = Get-UseGoLogs $useGoLogFile | Should -Exist $useGoLogContent = Get-Content $useGoLogFile -Raw - $useGoLogContent | Should -Match "Found tool in cache" + $useGoLogContent | Should -Match "Found in cache" } - Set-Location -Path "source" - $sourceLocation = Get-Location It "Run simple code" { $simpleLocation = Join-Path -Path $sourceLocation -ChildPath "simple" From e33ef4429ef208f75469ec404c8d83dacc31839e Mon Sep 17 00:00:00 2001 From: Nikita Bykov Date: Wed, 26 Aug 2020 15:58:05 +0300 Subject: [PATCH 2/2] resolved comments --- .github/workflows/build-go-packages.yml | 8 ++------ tests/Go.Tests.ps1 | 2 +- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build-go-packages.yml b/.github/workflows/build-go-packages.yml index 8f9b676..c5b6d3c 100644 --- a/.github/workflows/build-go-packages.yml +++ b/.github/workflows/build-go-packages.yml @@ -33,7 +33,6 @@ jobs: submodules: true - name: Build Go ${{ env.VERSION }} - shell: pwsh run: | ./builders/build-go.ps1 -Version $env:VERSION ` -Platform ${{ matrix.platform }} @@ -100,7 +99,6 @@ jobs: Write-Host "We need it because log of previous step 'Setup Go' is not available here yet." Write-Host "In testing step (Go.Tests.ps1) we analyze build log of 'Setup Go' task" Write-Host "to determine if Go.js version was consumed from cache and was downloaded" - Write-Host "Random values:" for ($i = 0; $i -lt 200; $i++) { Get-Random } - name: Run tests @@ -115,8 +113,6 @@ jobs: if: github.event.inputs.PUBLISH_RELEASES == 'true' needs: test_go runs-on: ubuntu-latest - outputs: - upload_url: ${{ steps.create_release.outputs.upload_url }} steps: - uses: actions/download-artifact@v2 @@ -129,7 +125,7 @@ jobs: tag_name: ${{ env.VERSION }}-${{ github.run_id }} release_name: ${{ env.VERSION }} body: | - Upload Go ${{ env.VERSION }} + Go ${{ env.VERSION }} - name: Upload release assets uses: actions/github-script@v2 @@ -155,7 +151,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Trigger "Create Pull Request" workflow - uses: actions/github-script@v2 + uses: actions/github-script@v3 with: github-token: ${{ secrets.PERSONAL_TOKEN }} script: | diff --git a/tests/Go.Tests.ps1 b/tests/Go.Tests.ps1 index 598cc02..c9ac579 100644 --- a/tests/Go.Tests.ps1 +++ b/tests/Go.Tests.ps1 @@ -4,6 +4,7 @@ Import-Module (Join-Path $PSScriptRoot "../helpers/common-helpers.psm1") BeforeAll { Set-Location -Path "source" $sourceLocation = Get-Location + function Get-UseGoLogs { # GitHub Windows images don't have `HOME` variable $homeDir = $env:HOME ?? $env:HOMEDRIVE @@ -17,7 +18,6 @@ BeforeAll { } } - Describe "Go" { It "is available" { "go version" | Should -ReturnZeroExitCode