build: stop uploading assets to S3 (#34116)

* build: remove S3 uploads

* build: remove ending slash upload-index-json (#34125)

Co-authored-by: Keeley Hammond <[email protected]>
Co-authored-by: Keeley Hammond <[email protected]>

.circleci/config.yml

@@ -14,6 +14,7 @@ parameters:
     type: boolean
     default: false
   
+  # TODO (vertedinde): migrate this variable to upload-to-az
   upload-to-s3:
     type: string
     default: '1'

script/lib/azput.js

@@ -20,12 +20,7 @@ let anErrorOccurred = false;
 function next (done) {
   const file = files.shift();
   if (!file) return done();
-  let key = filenameToKey(file);
-  // TODO: When we drop s3put, migrate the key to not include atom-shell in the callsites
-  key = key.replace('atom-shell/dist/', 'headers/dist/');
-  key = key.replace('atom-shell/symbols/', 'symbols/');
-  key = key.replace('atom-shell/tmp/', 'checksums-scratchpad/');
-  key = key.replace('electron-artifacts/', 'release-builds/');
+  const key = filenameToKey(file);
 
   const [containerName, ...keyPath] = key.split('/');
   const blobKey = keyPath.join('/');

script/lib/config.py

@@ -51,17 +51,6 @@ def get_env_var(name):
   return value
 
 
-def s3_config():
-  config = (get_env_var('S3_BUCKET'),
-            get_env_var('S3_ACCESS_KEY'),
-            get_env_var('S3_SECRET_KEY'))
-  message = ('Error: Please set the $ELECTRON_S3_BUCKET, '
-             '$ELECTRON_S3_ACCESS_KEY, and '
-             '$ELECTRON_S3_SECRET_KEY environment variables')
-  assert all(len(c) for c in config), message
-  return config
-
-
 def enable_verbose_mode():
   print('Running in verbose mode')
   global verbose_mode

script/lib/s3put.js

@@ -1,40 +0,0 @@
-/* eslint-disable camelcase */
-const AWS = require('aws-sdk');
-const fs = require('fs');
-const path = require('path');
-AWS.config.update({ region: 'us-west-2' });
-const s3 = new AWS.S3({ apiVersion: '2006-03-01' });
-
-const args = require('minimist')(process.argv.slice(2));
-
-let { bucket, prefix = '/', key_prefix = '', grant, _: files } = args;
-if (prefix && !prefix.endsWith(path.sep)) prefix = path.resolve(prefix) + path.sep;
-
-function filenameToKey (file) {
-  file = path.resolve(file);
-  if (file.startsWith(prefix)) file = file.substr(prefix.length - 1);
-  return key_prefix + (path.sep === '\\' ? file.replace(/\\/g, '/') : file);
-}
-
-let anErrorOccurred = false;
-function next (done) {
-  const file = files.shift();
-  if (!file) return done();
-  const key = filenameToKey(file);
-  console.log(`Uploading '${file}' to bucket '${bucket}' with key '${key}'...`);
-  s3.upload({
-    Bucket: bucket,
-    Key: key,
-    Body: fs.createReadStream(file),
-    ACL: grant
-  }, (err, data) => {
-    if (err) {
-      console.error(err);
-      anErrorOccurred = true;
-    }
-    next(done);
-  });
-}
-next(() => {
-  process.exit(anErrorOccurred ? 1 : 0);
-});

script/lib/util.py

@@ -15,7 +15,7 @@ except ImportError:
   from urllib2 import urlopen
 import zipfile
 
-from lib.config import is_verbose_mode, s3_config
+from lib.config import is_verbose_mode
 
 ELECTRON_DIR = os.path.abspath(
   os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
@@ -156,26 +156,9 @@ def get_electron_version():
     return 'v' + f.read().strip()
 
 def store_artifact(prefix, key_prefix, files):
-  # Legacy S3 Bucket
-  s3put(prefix, key_prefix, files)
-  # New AZ Storage
+  # Azure Storage
   azput(prefix, key_prefix, files)
 
-def s3put(prefix, key_prefix, files):
-  bucket, access_key, secret_key = s3_config()
-  env = os.environ.copy()
-  env['AWS_ACCESS_KEY_ID'] = access_key
-  env['AWS_SECRET_ACCESS_KEY'] = secret_key
-  output = execute([
-    'node',
-    os.path.join(os.path.dirname(__file__), 's3put.js'),
-    '--bucket', bucket,
-    '--prefix', prefix,
-    '--key_prefix', key_prefix,
-    '--grant', 'public-read',
-  ] + files, env)
-  print(output)
-
 def azput(prefix, key_prefix, files):
   env = os.environ.copy()
   output = execute([

script/release/release.js

@@ -78,8 +78,6 @@ async function validateReleaseAssets (release, validatingRelease) {
           console.log(`${fail} error verifyingShasums`, err);
         });
     }
-    const s3RemoteFiles = s3RemoteFilesForVersion(release.tag_name);
-    await verifyShasumsForRemoteFiles(s3RemoteFiles, true);
     const azRemoteFiles = azRemoteFilesForVersion(release.tag_name);
     await verifyShasumsForRemoteFiles(azRemoteFiles, true);
   }
@@ -195,15 +193,6 @@ const cloudStoreFilePaths = (version) => [
   'SHASUMS256.txt'
 ];
 
-function s3RemoteFilesForVersion (version) {
-  const bucket = 'https://gh-contractor-zcbenz.s3.amazonaws.com/';
-  const versionPrefix = `${bucket}atom-shell/dist/${version}/`;
-  return cloudStoreFilePaths(version).map((filePath) => ({
-    file: filePath,
-    url: `${versionPrefix}${filePath}`
-  }));
-}
-
 function azRemoteFilesForVersion (version) {
   const azCDN = 'https://artifacts.electronjs.org/headers/';
   const versionPrefix = `${azCDN}dist/${version}/`;

script/release/uploaders/upload-index-json.py

@@ -59,7 +59,7 @@ def main():
     with open(index_json, "wb") as f:
       f.write(new_content)
 
-    store_artifact(OUT_DIR, 'atom-shell/dist', [index_json])
+    store_artifact(OUT_DIR, 'headers/dist', [index_json])
 
 
 if __name__ == '__main__':

script/release/uploaders/upload-node-checksums.py

@@ -29,7 +29,7 @@ def main():
   ]
 
   if args.target_dir is None:
-    store_artifact(directory, 'atom-shell/dist/{0}'.format(args.version),
+    store_artifact(directory, 'headers/dist/{0}'.format(args.version),
                    checksums)
   else:
     copy_files(checksums, args.target_dir)

script/release/uploaders/upload-node-headers.py

@@ -45,9 +45,9 @@ def upload_node(version):
       versioned_header_tar = header_tar.format(version)
       shutil.copy2(generated_tar, os.path.join(GEN_DIR, versioned_header_tar))
 
-    store_artifact(GEN_DIR, 'atom-shell/dist/{0}'.format(version),
+    store_artifact(GEN_DIR, 'headers/dist/{0}'.format(version),
                    glob.glob('node-*.tar.gz'))
-    store_artifact(GEN_DIR, 'atom-shell/dist/{0}'.format(version),
+    store_artifact(GEN_DIR, 'headers/dist/{0}'.format(version),
                    glob.glob('iojs-*.tar.gz'))
 
   if PLATFORM == 'win32':
@@ -73,13 +73,13 @@ def upload_node(version):
     shutil.copy2(electron_lib, v4_node_lib)
 
     # Upload the node.lib.
-    store_artifact(DIST_DIR, 'atom-shell/dist/{0}'.format(version), [node_lib])
+    store_artifact(DIST_DIR, 'headers/dist/{0}'.format(version), [node_lib])
 
     # Upload the iojs.lib.
-    store_artifact(DIST_DIR, 'atom-shell/dist/{0}'.format(version), [iojs_lib])
+    store_artifact(DIST_DIR, 'headers/dist/{0}'.format(version), [iojs_lib])
 
     # Upload the v4 node.lib.
-    store_artifact(DIST_DIR, 'atom-shell/dist/{0}'.format(version),
+    store_artifact(DIST_DIR, 'headers/dist/{0}'.format(version),
                    [v4_node_lib])
 
 

script/release/uploaders/upload-symbols.py

@@ -56,7 +56,7 @@ def main():
 
   files += glob.glob(SYMBOLS_DIR + '/*/*/*.src.zip')
 
-  # The file upload needs to be atom-shell/symbols/:symbol_name/:hash/:symbol
+  # The file upload needs to be symbols/:symbol_name/:hash/:symbol
   os.chdir(SYMBOLS_DIR)
   files = [os.path.relpath(f, os.getcwd()) for f in files]
 
@@ -84,7 +84,7 @@ def run_symstore(pdb, dest, product):
 
 
 def upload_symbols(files):
-  store_artifact(SYMBOLS_DIR, 'atom-shell/symbols',
+  store_artifact(SYMBOLS_DIR, 'symbols',
         files)
 
 

script/release/uploaders/upload.py

@@ -343,8 +343,9 @@ def upload_electron(release, file_path, args):
     pass
 
   # if upload_to_s3 is set, skip github upload.
+  # todo (vertedinde): migrate this variable to upload_to_az
   if args.upload_to_s3:
-    key_prefix = 'electron-artifacts/{0}_{1}'.format(args.version,
+    key_prefix = 'release-builds/{0}_{1}'.format(args.version,
                                                      args.upload_timestamp)
     store_artifact(os.path.dirname(file_path), key_prefix, [file_path])
     upload_sha256_checksum(args.version, file_path, key_prefix)
@@ -369,7 +370,7 @@ def upload_io_to_github(release, filename, filepath, version):
 def upload_sha256_checksum(version, file_path, key_prefix=None):
   checksum_path = '{}.sha256sum'.format(file_path)
   if key_prefix is None:
-    key_prefix = 'atom-shell/tmp/{0}'.format(version)
+    key_prefix = 'checksums-scratchpad/{0}'.format(version)
   sha256 = hashlib.sha256()
   with open(file_path, 'rb') as f:
     sha256.update(f.read())