build: Updates to the release process (2-0-x) (#13615)

* Update to run new AppVeyor jobs

AppVeyor builds got split into two jobs, `electron-x64` and `electron-ia32`

(cherry picked from commit 90339b72604a0dcbfd3ff9f907c98203b3ee0e01)

* Move github release checking to node.js

Our upload script was relying on an old python script to find the GitHub release for uploading.

(cherry picked from commit 2040095b9644a21eef6627c3012f1f20d9f2a763)

script/ci-release-build.js

@@ -3,6 +3,11 @@ const request = require('request')
 const buildAppVeyorURL = 'https://windows-ci.electronjs.org/api/builds'
 const vstsURL = 'https://github.visualstudio.com/electron/_apis/build'
 
+const appVeyorJobs = {
+  'electron-x64': 'electron',
+  'electron-ia32': 'electron-39ng6'
+}
+
 const circleCIJobs = [
   'electron-linux-arm',
   'electron-linux-arm64',
@@ -71,9 +76,19 @@ async function circleCIcall (buildUrl, targetBranch, job, options) {
   console.log(`CircleCI release build request for ${job} successful.  Check ${circleResponse.build_url} for status.`)
 }
 
-async function buildAppVeyor (targetBranch, options) {
-  console.log(`Triggering AppVeyor to run build on branch: ${targetBranch} with release flag.`)
+function buildAppVeyor (targetBranch, options) {
   assert(process.env.APPVEYOR_TOKEN, 'APPVEYOR_TOKEN not found in environment')
+  const validJobs = Object.keys(appVeyorJobs)
+  if (options.job) {
+    assert(validJobs.includes(options.job), `Unknown AppVeyor CI job name: ${options.job}.  Valid values are: ${validJobs}.`)
+    callAppVeyor(targetBranch, options.job, options)
+  } else {
+    validJobs.forEach((job) => callAppVeyor(targetBranch, job, options))
+  }
+}
+
+async function callAppVeyor (targetBranch, job, options) {
+  console.log(`Triggering AppVeyor to run build job: ${job} on branch: ${targetBranch} with release flag.`)
   let environmentVariables = {}
 
   if (options.ghRelease) {
@@ -96,7 +111,7 @@ async function buildAppVeyor (targetBranch, options) {
     },
     body: JSON.stringify({
       accountName: 'AppVeyor',
-      projectSlug: 'electron',
+      projectSlug: appVeyorJobs[job],
       branch: targetBranch,
       environmentVariables
     }),
@@ -106,13 +121,13 @@ async function buildAppVeyor (targetBranch, options) {
     console.log('Error calling AppVeyor:', err)
   })
   const buildUrl = `https://windows-ci.electronjs.org/project/AppVeyor/electron/build/${appVeyorResponse.version}`
-  console.log(`AppVeyor release build request successful.  Check build status at ${buildUrl}`)
+  console.log(`AppVeyor release build request for ${job} successful.  Check build status at ${buildUrl}`)
 }
 
 function buildCircleCI (targetBranch, options) {
   const circleBuildUrl = `https://circleci.com/api/v1.1/project/github/electron/electron/tree/${targetBranch}?circle-token=${process.env.CIRCLE_TOKEN}`
   if (options.job) {
-    assert(circleCIJobs.includes(options.job), `Unknown CI job name: ${options.job}.`)
+    assert(circleCIJobs.includes(options.job), `Unknown CircleCI job name: ${options.job}. Valid values are: ${circleCIJobs}.`)
     circleCIcall(circleBuildUrl, targetBranch, options.job, options)
   } else {
     circleCIJobs.forEach((job) => circleCIcall(circleBuildUrl, targetBranch, job, options))
@@ -121,7 +136,7 @@ function buildCircleCI (targetBranch, options) {
 
 async function buildVSTS (targetBranch, options) {
   if (options.job) {
-    assert(vstsJobs.includes(options.job), `Unknown CI job name: ${options.job}.`)
+    assert(vstsJobs.includes(options.job), `Unknown VSTS CI job name: ${options.job}. Valid values are: ${vstsJobs}.`)
   }
   console.log(`Triggering VSTS to run build on branch: ${targetBranch} with release flag.`)
   assert(process.env.VSTS_TOKEN, 'VSTS_TOKEN not found in environment')

script/find-release.js

@@ -0,0 +1,37 @@
+const GitHub = require('github')
+const github = new GitHub()
+
+if (process.argv.length < 3) {
+  console.log('Usage: find-release version')
+  process.exit(1)
+}
+
+const version = process.argv[2]
+
+async function findRelease () {
+  github.authenticate({type: 'token', token: process.env.ELECTRON_GITHUB_TOKEN})
+  let releases = await github.repos.getReleases({
+    owner: 'electron',
+    repo: 'electron'
+  })
+  let targetRelease = releases.data.find(release => {
+    return release.tag_name === version
+  })
+  let returnObject = {}
+
+  if (targetRelease) {
+    returnObject = {
+      id: targetRelease.id,
+      draft: targetRelease.draft,
+      exists: true
+    }
+  } else {
+    returnObject = {
+      exists: false,
+      draft: false
+    }
+  }
+  console.log(JSON.stringify(returnObject))
+}
+
+findRelease()

script/lib/github.py

@@ -1,76 +0,0 @@
-#!/usr/bin/env python
-
-import json
-import os
-import re
-import sys
-
-REQUESTS_DIR = os.path.abspath(os.path.join(__file__, '..', '..', '..',
-                                            'vendor', 'requests'))
-sys.path.append(os.path.join(REQUESTS_DIR, 'build', 'lib'))
-sys.path.append(os.path.join(REQUESTS_DIR, 'build', 'lib.linux-x86_64-2.7'))
-import requests
-
-GITHUB_URL = 'https://api.github.com'
-GITHUB_UPLOAD_ASSET_URL = 'https://uploads.github.com'
-
-class GitHub:
-  def __init__(self, access_token):
-    self._authorization = 'token %s' % access_token
-
-    pattern = '^/repos/{0}/{0}/releases/{1}/assets$'.format('[^/]+', '[0-9]+')
-    self._releases_upload_api_pattern = re.compile(pattern)
-
-  def __getattr__(self, attr):
-    return _Callable(self, '/%s' % attr)
-
-  def send(self, method, path, **kw):
-    if not 'headers' in kw:
-      kw['headers'] = dict()
-    headers = kw['headers']
-    headers['Authorization'] = self._authorization
-    headers['Accept'] = 'application/vnd.github.manifold-preview'
-
-    # Switch to a different domain for the releases uploading API.
-    if self._releases_upload_api_pattern.match(path):
-      url = '%s%s' % (GITHUB_UPLOAD_ASSET_URL, path)
-    else:
-      url = '%s%s' % (GITHUB_URL, path)
-      # Data are sent in JSON format.
-      if 'data' in kw:
-        kw['data'] = json.dumps(kw['data'])
-
-    r = getattr(requests, method)(url, **kw).json()
-    if 'message' in r:
-      raise Exception(json.dumps(r, indent=2, separators=(',', ': ')))
-    return r
-
-
-class _Executable:
-  def __init__(self, gh, method, path):
-    self._gh = gh
-    self._method = method
-    self._path = path
-
-  def __call__(self, **kw):
-    return self._gh.send(self._method, self._path, **kw)
-
-
-class _Callable(object):
-  def __init__(self, gh, name):
-    self._gh = gh
-    self._name = name
-
-  def __call__(self, *args):
-    if len(args) == 0:
-      return self
-
-    name = '%s/%s' % (self._name, '/'.join([str(arg) for arg in args]))
-    return _Callable(self._gh, name)
-
-  def __getattr__(self, attr):
-    if attr in ['get', 'put', 'post', 'patch', 'delete']:
-      return _Executable(self._gh, attr, self._name)
-
-    name = '%s/%s' % (self._name, attr)
-    return _Callable(self._gh, name)

script/upload.py

@@ -1,8 +1,10 @@
 #!/usr/bin/env python
 
 import argparse
+import datetime
 import errno
 import hashlib
+import json
 import os
 import shutil
 import subprocess
@@ -14,7 +16,6 @@ from lib.config import PLATFORM, get_target_arch,  get_env_var, s3_config, \
                        get_zip_name
 from lib.util import electron_gyp, execute, get_electron_version, \
                      parse_version, scoped_cwd, s3put
-from lib.github import GitHub
 
 
 ELECTRON_REPO = 'electron/electron'
@@ -35,6 +36,9 @@ PDB_NAME = get_zip_name(PROJECT_NAME, ELECTRON_VERSION, 'pdb')
 
 def main():
   args = parse_args()
+  if  args.upload_to_s3:
+    utcnow = datetime.datetime.utcnow()
+    args.upload_timestamp = utcnow.strftime('%Y-%m-%d_%H:%M:%S')
 
   if not dist_newer_than_head():
     run_python_script('create-dist.py')
@@ -47,56 +51,40 @@ def main():
     sys.stderr.flush()
     return 1
 
-  github = GitHub(auth_token())
-  releases = github.repos(ELECTRON_REPO).releases.get()
   tag_exists = False
-  for r in releases:
-    if not r['draft'] and r['tag_name'] == args.version:
-      release = r
-      tag_exists = True
-      break
+  release = get_release(args.version)
+  if not release['draft']:
+    tag_exists = True
 
   if not args.upload_to_s3:
+    assert release['exists'], 'Release does not exist; cannot upload to GitHub!'
     assert tag_exists == args.overwrite, \
           'You have to pass --overwrite to overwrite a published release'
-    if not args.overwrite:
-      release = create_or_get_release_draft(github, releases, args.version,
-                                            tag_exists)
 
-  # Upload Electron with GitHub Releases API.
-  upload_electron(github, release, os.path.join(DIST_DIR, DIST_NAME),
-                  args.upload_to_s3)
+  # Upload Electron files.
+  upload_electron(release, os.path.join(DIST_DIR, DIST_NAME), args)
   if get_target_arch() != 'mips64el':
-    upload_electron(github, release, os.path.join(DIST_DIR, SYMBOLS_NAME),
-                    args.upload_to_s3)
+    upload_electron(release, os.path.join(DIST_DIR, SYMBOLS_NAME), args)
   if PLATFORM == 'darwin':
-    upload_electron(github, release, os.path.join(DIST_DIR,
-                    'electron-api.json'), args.upload_to_s3)
-    upload_electron(github, release, os.path.join(DIST_DIR, 'electron.d.ts'),
-                    args.upload_to_s3)
-    upload_electron(github, release, os.path.join(DIST_DIR, DSYM_NAME),
-                    args.upload_to_s3)
+    upload_electron(release, os.path.join(DIST_DIR, 'electron-api.json'), args)
+    upload_electron(release, os.path.join(DIST_DIR, 'electron.d.ts'), args)
+    upload_electron(release, os.path.join(DIST_DIR, DSYM_NAME), args)
   elif PLATFORM == 'win32':
-    upload_electron(github, release, os.path.join(DIST_DIR, PDB_NAME),
-                    args.upload_to_s3)
+    upload_electron(release, os.path.join(DIST_DIR, PDB_NAME), args)
 
   # Upload free version of ffmpeg.
   ffmpeg = get_zip_name('ffmpeg', ELECTRON_VERSION)
-  upload_electron(github, release, os.path.join(DIST_DIR, ffmpeg),
-                  args.upload_to_s3)
+  upload_electron(release, os.path.join(DIST_DIR, ffmpeg), args)
 
   chromedriver = get_zip_name('chromedriver', ELECTRON_VERSION)
-  upload_electron(github, release, os.path.join(DIST_DIR, chromedriver),
-                  args.upload_to_s3)
+  upload_electron(release, os.path.join(DIST_DIR, chromedriver), args)
   mksnapshot = get_zip_name('mksnapshot', ELECTRON_VERSION)
-  upload_electron(github, release, os.path.join(DIST_DIR, mksnapshot),
-                args.upload_to_s3)
+  upload_electron(release, os.path.join(DIST_DIR, mksnapshot), args)
 
   if get_target_arch().startswith('arm'):
     # Upload the x64 binary for arm/arm64 mksnapshot
     mksnapshot = get_zip_name('mksnapshot', ELECTRON_VERSION, 'x64')
-    upload_electron(github, release, os.path.join(DIST_DIR, mksnapshot),
-                    args.upload_to_s3)
+    upload_electron(release, os.path.join(DIST_DIR, mksnapshot), args)
 
   if PLATFORM == 'win32' and not tag_exists and not args.upload_to_s3:
     # Upload PDBs to Windows symbol server.
@@ -162,76 +150,26 @@ def dist_newer_than_head():
   return dist_time > int(head_time)
 
 
-def get_text_with_editor(name):
-  editor = os.environ.get('EDITOR', 'nano')
-  initial_message = '\n# Please enter the body of your release note for %s.' \
-                    % name
-
-  t = tempfile.NamedTemporaryFile(suffix='.tmp', delete=False)
-  t.write(initial_message)
-  t.close()
-  subprocess.call([editor, t.name])
-
-  text = ''
-  for line in open(t.name, 'r'):
-    if len(line) == 0 or line[0] != '#':
-      text += line
-
-  os.unlink(t.name)
-  return text
-
-def create_or_get_release_draft(github, releases, tag, tag_exists):
-  # Search for existing draft.
-  for release in releases:
-    if release['draft'] and release['tag_name'] == tag:
-      return release
-
-  if tag_exists:
-    tag = 'do-not-publish-me'
-  return create_release_draft(github, tag)
-
-
-def create_release_draft(github, tag):
-  name = '{0} {1} beta'.format(PROJECT_NAME, tag)
-  if os.environ.has_key('CI'):
-    body = '(placeholder)'
-  else:
-    body = get_text_with_editor(name)
-  if body == '':
-    sys.stderr.write('Quit due to empty release note.\n')
-    sys.exit(0)
-
-  data = dict(tag_name=tag, name=name, body=body, draft=True, prerelease=True)
-  r = github.repos(ELECTRON_REPO).releases.post(data=data)
-  return r
-
-
-def upload_electron(github, release, file_path, upload_to_s3):
+def upload_electron(release, file_path, args):
+  filename = os.path.basename(file_path)
 
   # if upload_to_s3 is set, skip github upload.
-  if upload_to_s3:
+  if args.upload_to_s3:
     bucket, access_key, secret_key = s3_config()
-    key_prefix = 'electron-artifacts/{0}'.format(release['tag_name'])
+    key_prefix = 'electron-artifacts/{0}_{1}'.format(args.version,
+                                                     args.upload_timestamp)
     s3put(bucket, access_key, secret_key, os.path.dirname(file_path),
           key_prefix, [file_path])
-    upload_sha256_checksum(release['tag_name'], file_path, key_prefix)
+    upload_sha256_checksum(args.version, file_path, key_prefix)
+    s3url = 'https://gh-contractor-zcbenz.s3.amazonaws.com'
+    print '{0} uploaded to {1}/{2}/{0}'.format(filename, s3url, key_prefix)
     return
 
-  # Delete the original file before uploading in CI.
-  filename = os.path.basename(file_path)
-  if os.environ.has_key('CI'):
-    try:
-      for asset in release['assets']:
-        if asset['name'] == filename:
-          github.repos(ELECTRON_REPO).releases.assets(asset['id']).delete()
-    except Exception:
-      pass
-
   # Upload the file.
   upload_io_to_github(release, filename, file_path)
 
   # Upload the checksum file.
-  upload_sha256_checksum(release['tag_name'], file_path)
+  upload_sha256_checksum(args.version, file_path)
 
   # Upload ARM assets without the v7l suffix for backwards compatibility
   # TODO Remove for 2.0
@@ -239,7 +177,7 @@ def upload_electron(github, release, file_path, upload_to_s3):
     arm_filename = filename.replace('armv7l', 'arm')
     arm_file_path = os.path.join(os.path.dirname(file_path), arm_filename)
     shutil.copy2(file_path, arm_file_path)
-    upload_electron(github, release, arm_file_path, upload_to_s3)
+    upload_electron(release, arm_file_path, args)
 
 
 def upload_io_to_github(release, filename, filepath):
@@ -273,6 +211,12 @@ def auth_token():
   return token
 
 
+def get_release(version):
+  script_path = os.path.join(SOURCE_ROOT, 'script', 'find-release.js')
+  release_info = execute(['node', script_path, version])
+  release = json.loads(release_info)
+  return release
+
 if __name__ == '__main__':
   import sys
   sys.exit(main())