Browse Source

ci: update to CircleCI v2 config and APIs (#22310)

* ci: Update to CircleCI v2 config and APIs

This reverts commit e6807b387c9240dc4516c0fcaa5509db89bc5bda.

* build: update release build endpoint from /jobs to /job (#21232)


(cherry picked from commit 41f1569c465c85cb20edd34b95a69b25829e1a53)

* chore: workflows and pipeline state were split in the circle API (#21441)


(cherry picked from commit ec0edb757ae1734b48f4c918edfdce25d51dec15)

Co-authored-by: Samuel Attard <[email protected]>
John Kleinschmidt 5 years ago
parent
commit
0e8aa5c252
2 changed files with 236 additions and 35 deletions
  1. 116 16
      .circleci/config.yml
  2. 120 19
      script/release/ci-release-build.js

+ 116 - 16
.circleci/config.yml

@@ -1,3 +1,46 @@
+version: 2.1
+
+parameters:
+  upload-to-s3:
+    type: string
+    default: '1'
+  
+  run-lint:
+    type: boolean
+    default: true
+
+  run-build-linux:
+    type: boolean
+    default: true
+
+  run-build-mac:
+    type: boolean
+    default: true
+
+  run-linux-x64-publish:
+    type: boolean
+    default: false
+
+  run-linux-ia32-publish:
+    type: boolean
+    default: false
+
+  run-linux-arm-publish:
+    type: boolean
+    default: false
+  
+  run-linux-arm64-publish:
+    type: boolean
+    default: false
+  
+  run-osx-publish:
+    type: boolean
+    default: false
+
+  run-mas-publish:
+    type: boolean
+    default: false
+
 # The config expects the following environment variables to be set:
 #  - "SLACK_WEBHOOK" Slack hook URL to send notifications.
 #
@@ -674,7 +717,7 @@ steps-lint: &steps-lint
           chromium_revision="$(grep -A1 chromium_version src/electron/DEPS | tr -d '\n' | cut -d\' -f4)"
           gn_version="$(curl -sL "https://chromium.googlesource.com/chromium/src/+/${chromium_revision}/DEPS?format=TEXT" | base64 -d | grep gn_version | head -n1 | cut -d\' -f4)"
 
-          cipd ensure -ensure-file - -root . <<-CIPD
+          cipd ensure -ensure-file - -root . \<<-CIPD
           \$ServiceURL https://chrome-infra-packages.appspot.com/
           @Subdir src/buildtools/linux64
           gn/gn/linux-amd64 $gn_version
@@ -1028,7 +1071,6 @@ chromium-upgrade-branches: &chromium-upgrade-branches
   /chromium\-upgrade\/[0-9]+/
 
 # List of all jobs.
-version: 2
 jobs:
   # Layer 0: Lint. Standalone.
   lint:
@@ -1148,6 +1190,8 @@ jobs:
       <<: *env-linux-2xlarge-release
       GCLIENT_EXTRA_ARGS: '--custom-var=checkout_boto=True --custom-var=checkout_requests=True'
       <<: *env-release-build
+      <<: *env-enable-sccache
+      UPLOAD_TO_S3: << pipeline.parameters.upload-to-s3 >>
     <<: *steps-electron-build-for-publish
 
   linux-ia32-debug:
@@ -1198,6 +1242,8 @@ jobs:
       GCLIENT_EXTRA_ARGS: '--custom-var=checkout_boto=True --custom-var=checkout_requests=True'
       <<: *env-ia32
       <<: *env-release-build
+      <<: *env-enable-sccache
+      UPLOAD_TO_S3: << pipeline.parameters.upload-to-s3 >>
     <<: *steps-electron-build-for-publish
 
   linux-arm-debug:
@@ -1248,7 +1294,9 @@ jobs:
       <<: *env-linux-2xlarge-release
       <<: *env-arm
       <<: *env-release-build
+      <<: *env-enable-sccache
       GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_boto=True --custom-var=checkout_requests=True'
+      UPLOAD_TO_S3: << pipeline.parameters.upload-to-s3 >>
     <<: *steps-electron-build-for-publish
 
   linux-arm64-debug:
@@ -1315,7 +1363,9 @@ jobs:
       <<: *env-linux-2xlarge-release
       <<: *env-arm64
       <<: *env-release-build
+      <<: *env-enable-sccache
       GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm64=True --custom-var=checkout_boto=True --custom-var=checkout_requests=True'
+      UPLOAD_TO_S3: << pipeline.parameters.upload-to-s3 >>
     <<: *steps-electron-build-for-publish
 
   osx-testing:
@@ -1373,7 +1423,9 @@ jobs:
     environment:
       <<: *env-mac-large-release
       <<: *env-release-build
+      <<: *env-enable-sccache
       GCLIENT_EXTRA_ARGS: '--custom-var=checkout_boto=True --custom-var=checkout_requests=True'
+      UPLOAD_TO_S3: << pipeline.parameters.upload-to-s3 >>
     <<: *steps-electron-build-for-publish
 
   mas-testing:
@@ -1437,7 +1489,9 @@ jobs:
       <<: *env-mac-large-release
       <<: *env-mas
       <<: *env-release-build
+      <<: *env-enable-sccache
       GCLIENT_EXTRA_ARGS: '--custom-var=checkout_boto=True --custom-var=checkout_requests=True'
+      UPLOAD_TO_S3: << pipeline.parameters.upload-to-s3 >>
     <<: *steps-electron-build-for-publish
 
   # Layer 3: Tests.
@@ -1693,11 +1747,56 @@ jobs:
 
 workflows:
   version: 2
+
+  # The publish workflows below each contain one job so that they are 
+  # compatible with how sudowoodo works today.  If these workflows are
+  # changed to have multiple jobs, then scripts/release/ci-release-build.js
+  # will need to be updated and there will most likely need to be changes to
+  # sudowoodo
+
+  publish-x64-linux:
+    when: << pipeline.parameters.run-linux-x64-publish >>
+    jobs:
+    - linux-x64-publish:
+        context: release-env
+
+  publish-ia32-linux:
+    when: << pipeline.parameters.run-linux-ia32-publish >>
+    jobs:
+    - linux-ia32-publish:
+        context: release-env
+
+  publish-arm-linux:
+    when: << pipeline.parameters.run-linux-arm-publish >>
+    jobs:
+    - linux-arm-publish:
+        context: release-env
+  
+  publish-arm64-linux:
+    when: << pipeline.parameters.run-linux-arm64-publish >>
+    jobs:
+    - linux-arm64-publish:
+        context: release-env
+  
+  publish-osx:
+    when: << pipeline.parameters.run-osx-publish >>
+    jobs:
+    - osx-publish:
+        context: release-env
+
+  publish-mas:
+    when: << pipeline.parameters.run-mas-publish >>
+    jobs:
+    - mas-publish:
+        context: release-env
+
   lint:
+    when: << pipeline.parameters.run-lint >>
     jobs:
       - lint
 
   build-linux:
+    when: << pipeline.parameters.run-build-linux >>
     jobs:
       - linux-checkout-fast
       - linux-checkout-and-save-cache
@@ -1764,6 +1863,7 @@ workflows:
             - linux-checkout-fast
 
   build-mac:
+    when: << pipeline.parameters.run-build-mac >>
     jobs:
       - mac-checkout-fast
       - mac-checkout-and-save-cache
@@ -1818,11 +1918,11 @@ workflows:
                 - master
                 - *chromium-upgrade-branches
     jobs:
-      - linux-checkout
+      - linux-checkout-fast
 
       - linux-x64-release:
           requires:
-            - linux-checkout
+            - linux-checkout-fast
       - linux-x64-release-tests:
           requires:
             - linux-x64-release
@@ -1834,7 +1934,7 @@ workflows:
             - linux-x64-release
       - linux-x64-chromedriver:
           requires:
-            - linux-checkout
+            - linux-checkout-fast
       - linux-x64-release-summary:
           requires:
             - linux-x64-release
@@ -1844,7 +1944,7 @@ workflows:
 
       - linux-ia32-release:
           requires:
-            - linux-checkout
+            - linux-checkout-fast
       - linux-ia32-release-tests:
           requires:
             - linux-ia32-release
@@ -1856,7 +1956,7 @@ workflows:
             - linux-ia32-release
       - linux-ia32-chromedriver:
           requires:
-            - linux-checkout
+            - linux-checkout-fast
       - linux-ia32-release-summary:
           requires:
             - linux-ia32-release
@@ -1866,10 +1966,10 @@ workflows:
 
       - linux-arm-release:
           requires:
-            - linux-checkout
+            - linux-checkout-fast
       - linux-arm-chromedriver:
           requires:
-            - linux-checkout
+            - linux-checkout-fast
       - linux-arm-release-summary:
           requires:
             - linux-arm-release
@@ -1878,10 +1978,10 @@ workflows:
 
       - linux-arm64-release:
           requires:
-            - linux-checkout
+            - linux-checkout-fast
       - linux-arm64-chromedriver:
           requires:
-            - linux-checkout
+            - linux-checkout-fast
       - linux-arm64-release-summary:
           requires:
             - linux-arm64-release
@@ -1897,11 +1997,11 @@ workflows:
                 - master
                 - *chromium-upgrade-branches
     jobs:
-      - mac-checkout
+      - mac-checkout-fast
 
       - osx-release:
           requires:
-            - mac-checkout
+            - mac-checkout-fast
       - osx-release-tests:
           requires:
             - osx-release
@@ -1913,7 +2013,7 @@ workflows:
             - osx-release
       - osx-chromedriver:
           requires:
-            - mac-checkout
+            - mac-checkout-fast
       - osx-release-summary:
           requires:
           - osx-release
@@ -1923,7 +2023,7 @@ workflows:
 
       - mas-release:
           requires:
-            - mac-checkout
+            - mac-checkout-fast
       - mas-release-tests:
           requires:
             - mas-release
@@ -1935,7 +2035,7 @@ workflows:
             - mas-release
       - mas-chromedriver:
           requires:
-            - mac-checkout
+            - mac-checkout-fast
       - mas-release-summary:
           requires:
           - mas-release

+ 120 - 19
script/release/ci-release-build.js

@@ -2,8 +2,11 @@ if (!process.env.CI) require('dotenv-safe').load()
 
 const assert = require('assert')
 const request = require('request')
-const buildAppVeyorURL = 'https://ci.appveyor.com/api/builds'
-const vstsURL = 'https://github.visualstudio.com/electron/_apis/build'
+
+const BUILD_APPVEYOR_URL = 'https://ci.appveyor.com/api/builds'
+const CIRCLECI_PIPELINE_URL = 'https://circleci.com/api/v2/project/gh/electron/electron/pipeline'
+const VSTS_URL = 'https://github.visualstudio.com/electron/_apis/build'
+const CIRCLECI_WAIT_TIME = process.env.CIRCLECI_WAIT_TIME || 30000
 
 const appVeyorJobs = {
   'electron-x64': 'electron-x64-release',
@@ -55,30 +58,129 @@ async function makeRequest (requestOptions, parseResponse) {
   })
 }
 
-async function circleCIcall (buildUrl, targetBranch, job, options) {
+async function circleCIcall (targetBranch, job, options) {
   console.log(`Triggering CircleCI to run build job: ${job} on branch: ${targetBranch} with release flag.`)
   const buildRequest = {
-    'build_parameters': {
-      'CIRCLE_JOB': job
+    'branch': targetBranch,
+    'parameters': {
+      'run-lint': false,
+      'run-build-linux': false,
+      'run-build-mac': false
     }
   }
-
-  if (!options.ghRelease) {
-    buildRequest.build_parameters.UPLOAD_TO_S3 = 1
+  if (options.ghRelease) {
+    buildRequest.parameters['upload-to-s3'] = '0'
+  } else {
+    buildRequest.parameters['upload-to-s3'] = '1'
   }
+  buildRequest.parameters[`run-${job}`] = true
   jobRequestedCount++
-  const circleResponse = await makeRequest({
-    method: 'POST',
-    url: buildUrl,
+  // The logic below expects that the CircleCI workflows for releases each
+  // contain only one job in order to maintain compatibility with sudowoodo.
+  // If the workflows are changed in the CircleCI config.yml, this logic will
+  // also need to be changed as well as possibly changing sudowoodo.
+  try {
+    const circleResponse = await circleCIRequest(CIRCLECI_PIPELINE_URL, 'POST', buildRequest)
+    console.log(`CircleCI release build pipeline ${circleResponse.id} for ${job} triggered.`)
+    const pipelineInfoUrl = `https://circleci.com/api/v2/pipeline/${circleResponse.id}`
+    const workflowId = await getCircleCIWorkflowId(circleResponse.id)
+    if (workflowId === -1) {
+      return
+    }
+    console.log(`CircleCI release build workflow running at https://circleci.com/workflow-run/${workflowId} for ${job}.`)
+    const jobNumber = await getCircleCIJobNumber(workflowId)
+    if (jobNumber === -1) {
+      return
+    }
+    const jobUrl = `https://circleci.com/gh/electron/electron/${jobNumber}`
+    console.log(`CircleCI release build request for ${job} successful.  Check ${jobUrl} for status.`)
+  } catch (err) {
+    console.log('Error calling CircleCI: ', err)
+  }
+}
+
+async function getCircleCIWorkflowId (pipelineId) {
+  const pipelineInfoUrl = `https://circleci.com/api/v2/pipeline/${pipelineId}`
+  let workflowId = 0
+  while (workflowId === 0) {
+    const pipelineInfo = await circleCIRequest(pipelineInfoUrl, 'GET')
+    switch (pipelineInfo.state) {
+      case 'created': {
+        const workflows = await circleCIRequest(`${pipelineInfoUrl}/workflow`, 'GET')
+        if (workflows.items.length === 1) {
+          workflowId = workflows.items[0].id
+          break
+        }
+        console.log('Unxpected number of workflows, response was:', pipelineInfo)
+        workflowId = -1
+        break
+      }
+      case 'error': {
+        console.log('Error retrieving workflows, response was:', pipelineInfo)
+        workflowId = -1
+        break
+      }
+    }
+    await new Promise(resolve => setTimeout(resolve, CIRCLECI_WAIT_TIME))
+  }
+  return workflowId
+}
+
+async function getCircleCIJobNumber (workflowId) {
+  const jobInfoUrl = `https://circleci.com/api/v2/workflow/${workflowId}/job`
+  let jobNumber = 0
+  while (jobNumber === 0) {
+    const jobInfo = await circleCIRequest(jobInfoUrl, 'GET')
+    if (!jobInfo.items) {
+      continue
+    }
+    if (jobInfo.items.length !== 1) {
+      console.log('Unxpected number of jobs, response was:', jobInfo)
+      jobNumber = -1
+      break
+    }
+
+    switch (jobInfo.items[0].status) {
+      case 'not_running':
+      case 'queued':
+      case 'running': {
+        if (jobInfo.items[0].job_number && !isNaN(jobInfo.items[0].job_number)) {
+          jobNumber = jobInfo.items[0].job_number
+        }
+        break
+      }
+      case 'canceled':
+      case 'error':
+      case 'infrastructure_fail':
+      case 'timedout':
+      case 'not_run':
+      case 'failed': {
+        console.log(`Error job returned a status of ${jobInfo.items[0].status}, response was:`, jobInfo)
+        jobNumber = -1
+        break
+      }
+    }
+    await new Promise(resolve => setTimeout(resolve, CIRCLECI_WAIT_TIME))
+  }
+  return jobNumber
+}
+
+async function circleCIRequest (url, method, requestBody) {
+  return makeRequest({
+    auth: {
+      username: process.env.CIRCLE_TOKEN,
+      password: ''
+    },
+    method,
+    url,
     headers: {
       'Content-Type': 'application/json',
       'Accept': 'application/json'
     },
-    body: JSON.stringify(buildRequest)
+    body: requestBody ? JSON.stringify(requestBody) : null
   }, true).catch(err => {
     console.log('Error calling CircleCI:', err)
   })
-  console.log(`CircleCI release build request for ${job} successful.  Check ${circleResponse.build_url} for status.`)
 }
 
 function buildAppVeyor (targetBranch, options) {
@@ -102,7 +204,7 @@ async function callAppVeyor (targetBranch, job, options) {
   }
 
   const requestOpts = {
-    url: buildAppVeyorURL,
+    url: BUILD_APPVEYOR_URL,
     auth: {
       bearer: process.env.APPVEYOR_CLOUD_TOKEN
     },
@@ -126,12 +228,11 @@ async function callAppVeyor (targetBranch, job, options) {
 }
 
 function buildCircleCI (targetBranch, options) {
-  const circleBuildUrl = `https://circleci.com/api/v1.1/project/github/electron/electron/tree/${targetBranch}?circle-token=${process.env.CIRCLE_TOKEN}`
   if (options.job) {
     assert(circleCIJobs.includes(options.job), `Unknown CircleCI job name: ${options.job}. Valid values are: ${circleCIJobs}.`)
-    circleCIcall(circleBuildUrl, targetBranch, options.job, options)
+    circleCIcall(targetBranch, options.job, options)
   } else {
-    circleCIJobs.forEach((job) => circleCIcall(circleBuildUrl, targetBranch, job, options))
+    circleCIJobs.forEach((job) => circleCIcall(targetBranch, job, options))
   }
 }
 
@@ -158,7 +259,7 @@ async function buildVSTS (targetBranch, options) {
   }
 
   const requestOpts = {
-    url: `${vstsURL}/definitions?api-version=4.1`,
+    url: `${VSTS_URL}/definitions?api-version=4.1`,
     auth: {
       user: '',
       password: process.env.VSTS_TOKEN
@@ -184,7 +285,7 @@ async function callVSTSBuild (build, targetBranch, environmentVariables) {
     buildBody.parameters = JSON.stringify(environmentVariables)
   }
   const requestOpts = {
-    url: `${vstsURL}/builds?api-version=4.1`,
+    url: `${VSTS_URL}/builds?api-version=4.1`,
     auth: {
       user: '',
       password: process.env.VSTS_TOKEN