build: improve circleci config (#33944)

* build: improve circleci config (#33881)

* build: fix conditional restore of git cache

* build: split lint out of setup.yml

* ci: update release script to handle new CircleCI configs (#33914)

Co-authored-by: John Kleinschmidt <[email protected]>

.circleci/.gitignore

@@ -0,0 +1 @@
+config-staging

.circleci/config.yml

@@ -6,6 +6,7 @@ setup: true
 # Orbs
 orbs:
   path-filtering: circleci/[email protected]
+  continuation: circleci/[email protected]
 
 # All input parameters to pass to build config
 parameters:
@@ -43,103 +44,33 @@ parameters:
     default: all
     enum: ["all", "osx-x64", "osx-arm64", "mas-x64", "mas-arm64"]
 
-# Envs
-env-global: &env-global
-  ELECTRON_OUT_DIR: Default
-
-env-linux-medium: &env-linux-medium
-  <<: *env-global
-  NUMBER_OF_NINJA_PROCESSES: 3
-
-# Executors
-executors:
-  linux-docker:
-    parameters:
-      size:
-        description: "Docker executor size"
-        default: 2xlarge+
-        type: enum
-        enum: ["medium", "xlarge", "2xlarge+"]
-    docker:
-      - image: ghcr.io/electron/build:27db4a3e3512bfd2e47f58cea69922da0835f1d9
-    resource_class: << parameters.size >>
-
-# List of always run steps
-step-checkout-electron: &step-checkout-electron
-  checkout:
-    path: src/electron
-
-steps-lint: &steps-lint
-  steps:
-    - *step-checkout-electron
-    - run:
-        name: Setup third_party Depot Tools
-        command: |
-          # "depot_tools" has to be checkout into "//third_party/depot_tools" so pylint.py can a "pylintrc" file.
-          git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git src/third_party/depot_tools
-          echo 'export PATH="$PATH:'"$PWD"'/src/third_party/depot_tools"' >> $BASH_ENV
-    - run:
-        name: Download GN Binary
-        command: |
-          chromium_revision="$(grep -A1 chromium_version src/electron/DEPS | tr -d '\n' | cut -d\' -f4)"
-          gn_version="$(curl -sL "https://chromium.googlesource.com/chromium/src/+/${chromium_revision}/DEPS?format=TEXT" | base64 -d | grep gn_version | head -n1 | cut -d\' -f4)"
-
-          cipd ensure -ensure-file - -root . \<<-CIPD
-          \$ServiceURL https://chrome-infra-packages.appspot.com/
-          @Subdir src/buildtools/linux64
-          gn/gn/linux-amd64 $gn_version
-          CIPD
-
-          echo 'export CHROMIUM_BUILDTOOLS_PATH="'"$PWD"'/src/buildtools"' >> $BASH_ENV
-    - run:
-        name: Download clang-format Binary
-        command: |
-          chromium_revision="$(grep -A1 chromium_version src/electron/DEPS | tr -d '\n' | cut -d\' -f4)"
-
-          sha1_path='buildtools/linux64/clang-format.sha1'
-          curl -sL "https://chromium.googlesource.com/chromium/src/+/${chromium_revision}/${sha1_path}?format=TEXT" | base64 -d > "src/${sha1_path}"
-
-          download_from_google_storage.py --no_resume --no_auth --bucket chromium-clang-format -s "src/${sha1_path}"
-    - run:
-        name: Run Lint
-        command: |
-          # gn.py tries to find a gclient root folder starting from the current dir.
-          # When it fails and returns "None" path, the whole script fails. Let's "fix" it.
-          touch .gclient
-          # Another option would be to checkout "buildtools" inside the Electron checkout,
-          # but then we would lint its contents (at least gn format), and it doesn't pass it.
-
-          cd src/electron
-          node script/yarn install --frozen-lockfile
-          node script/yarn lint
-    - run:
-        name: Run Script Typechecker
-        command: |
-          cd src/electron
-          node script/yarn tsc -p tsconfig.script.json
-
-# List of always run jobs.
 jobs:
-  lint:
-    executor:
-      name: linux-docker
-      size: medium
-    environment:
-      <<: *env-linux-medium
-    <<: *steps-lint
-
-# Initial setup workflow
-workflows:
-  lint:
-    jobs:
-      # Job inherited from path-filtering orb
-      - path-filtering/filter:
+  generate-config:
+    docker:
+      - image: cimg/node:16.14
+    steps:
+      - checkout
+      - path-filtering/set-parameters:
           base-revision: main
-          # Params for mapping; `path-to-test parameter-to-set value-for-parameter` for each row
           mapping: |
             ^((?!docs/).)*$ run-build-mac true
             ^((?!docs/).)*$ run-build-linux true
             docs/.* run-docs-only true
             ^((?!docs/).)*$ run-docs-only false
-          config-path: .circleci/build_config.yml
-      - lint
+      - run:
+          command: |
+            cd .circleci/config
+            yarn
+            export CIRCLECI_BINARY="$HOME/circleci"
+            curl -fLSs https://raw.githubusercontent.com/CircleCI-Public/circleci-cli/master/install.sh | DESTDIR=$CIRCLECI_BINARY bash
+            node build.js
+          name: Pack config.yml
+      - continuation/continue:
+          configuration_path: .circleci/config-staging/built.yml
+          parameters: /tmp/pipeline-parameters.json
+
+# Initial setup workflow
+workflows:
+  setup:
+    jobs:
+      - generate-config

.circleci/build_config.yml → .circleci/config/base.yml

@@ -890,12 +890,12 @@ step-touch-sync-done: &step-touch-sync-done
 step-maybe-restore-src-cache: &step-maybe-restore-src-cache
   restore_cache:
     keys:
-      - v8-src-cache-{{ checksum "src/electron/.depshash" }}
+      - v14-src-cache-{{ checksum "src/electron/.depshash" }}
     name: Restoring src cache
 step-maybe-restore-src-cache-marker: &step-maybe-restore-src-cache-marker
   restore_cache:
     keys:
-      - v1-src-cache-marker-{{ checksum "src/electron/.depshash" }}
+      - v14-src-cache-marker-{{ checksum "src/electron/.depshash" }}
     name: Restoring src cache marker
 
 # Restore exact or closest git cache based on the hash of DEPS and .circle-sync-done
@@ -904,10 +904,10 @@ step-maybe-restore-src-cache-marker: &step-maybe-restore-src-cache-marker
 step-maybe-restore-git-cache: &step-maybe-restore-git-cache
   restore_cache:
     paths:
-      - ~/.gclient-cache
+      - git-cache
     keys:
-      - v2-gclient-cache-{{ checksum "src/electron/.circle-sync-done" }}-{{ checksum "src/electron/DEPS" }}
-      - v2-gclient-cache-{{ checksum "src/electron/.circle-sync-done" }}
+      - v1-git-cache-{{ checksum "src/electron/.circle-sync-done" }}-{{ checksum "src/electron/DEPS" }}
+      - v1-git-cache-{{ checksum "src/electron/.circle-sync-done" }}
     name: Conditionally restoring git cache
 
 step-restore-out-cache: &step-restore-out-cache
@@ -924,15 +924,15 @@ step-set-git-cache-path: &step-set-git-cache-path
     command: |
       # CircleCI does not support interpolation when setting environment variables.
       # https://circleci.com/docs/2.0/env-vars/#setting-an-environment-variable-in-a-shell-command
-      echo 'export GIT_CACHE_PATH="$HOME/.gclient-cache"' >> $BASH_ENV
+      echo 'export GIT_CACHE_PATH="$PWD/git-cache"' >> $BASH_ENV
 
 # Persist the git cache based on the hash of DEPS and .circle-sync-done
 # If the src cache was restored above then this will persist an empty cache
 step-save-git-cache: &step-save-git-cache
   save_cache:
     paths:
-      - ~/.gclient-cache
-    key: v2-gclient-cache-{{ checksum "src/electron/.circle-sync-done" }}-{{ checksum "src/electron/DEPS" }}
+      - git-cache
+    key: v1-git-cache-{{ checksum "src/electron/.circle-sync-done" }}-{{ checksum "src/electron/DEPS" }}
     name: Persisting git cache
 
 step-save-out-cache: &step-save-out-cache
@@ -977,7 +977,7 @@ step-save-src-cache: &step-save-src-cache
   save_cache:
     paths:
       - /var/portal
-    key: v8-src-cache-{{ checksum "/var/portal/src/electron/.depshash" }}
+    key: v14-src-cache-{{ checksum "/var/portal/src/electron/.depshash" }}
     name: Persisting src cache
 step-make-src-cache-marker: &step-make-src-cache-marker
   run:
@@ -987,7 +987,7 @@ step-save-src-cache-marker: &step-save-src-cache-marker
   save_cache:
     paths:
       - .src-cache-marker
-    key: v1-src-cache-marker-{{ checksum "/var/portal/src/electron/.depshash" }}
+    key: v14-src-cache-marker-{{ checksum "/var/portal/src/electron/.depshash" }}
 
 step-maybe-early-exit-no-doc-change: &step-maybe-early-exit-no-doc-change
   run:
@@ -1363,6 +1363,39 @@ commands:
             - *step-checkout-electron
             - *step-run-electron-only-hooks
             - *step-generate-deps-hash-cleanly
+            - when:
+                condition: << parameters.save-git-cache >>
+                steps:
+                  - *step-save-git-cache
+            # Mark sync as done _after_ saving the git cache so that it is uploaded
+            # only when the src cache was not present
+            # Their are theoretically two cases for this cache key
+            # 1. `vX-git-cache-DONE-{deps_hash}
+            # 2. `vX-git-cache-EMPTY-{deps_hash}
+            #
+            # Case (1) occurs when the flag file has "DONE" in it
+            # which only occurs when "step-mark-sync-done" is run
+            # or when the src cache was restored successfully as that
+            # flag file contains "DONE" in the src cache.
+            #
+            # Case (2) occurs when the flag file is empty, this occurs
+            # when the src cache was not restored and "step-mark-sync-done"
+            # has not run yet.
+            #
+            # Notably both of these cases also have completely different
+            # gclient cache states.
+            # In (1) the git cache is completely empty as we didn't run
+            # "gclient sync" because the src cache was restored.
+            # In (2) the git cache is full as we had to run "gclient sync"
+            #
+            # This allows us to do make the follow transitive assumption:
+            # In cases where the src cache is restored, saving the git cache
+            # will save an empty cache.  In cases where the src cache is built
+            # during this build the git cache will save a full cache.
+            #
+            # In order words if there is a src cache for a given DEPS hash
+            # the git cache restored will be empty.  But if the src cache
+            # is missing we will restore a useful git cache.
             - *step-mark-sync-done
             - *step-minimize-workspace-size-from-checkout
             - *step-delete-git-directories
@@ -2418,3 +2451,6 @@ workflows:
               ignore: /pull\/[0-9]+/
           requires:
             - mas-testing-arm64
+  lint:
+    jobs:
+      - lint

.circleci/config/build.js

@@ -0,0 +1,34 @@
+const cp = require('child_process');
+const fs = require('fs-extra');
+const path = require('path');
+const yaml = require('js-yaml');
+
+const STAGING_DIR = path.resolve(__dirname, '..', 'config-staging');
+
+function copyAndExpand(dir = './') {
+    const absDir = path.resolve(__dirname, dir);
+    const targetDir = path.resolve(STAGING_DIR, dir);
+
+    if (!fs.existsSync(targetDir)) {
+        fs.mkdirSync(targetDir);
+    }
+
+    for (const file of fs.readdirSync(absDir)) {
+        if (!file.endsWith('.yml')) {
+            if (fs.statSync(path.resolve(absDir, file)).isDirectory()) {
+                copyAndExpand(path.join(dir, file));
+            }
+            continue;
+        }
+
+        fs.writeFileSync(path.resolve(targetDir, file), yaml.dump(yaml.load(fs.readFileSync(path.resolve(absDir, file), 'utf8')), {
+            noRefs: true,
+        }));
+    }
+}
+
+if (fs.pathExists(STAGING_DIR)) fs.removeSync(STAGING_DIR);
+copyAndExpand();
+
+const output = cp.spawnSync(process.env.CIRCLECI_BINARY || 'circleci', ['config', 'pack', STAGING_DIR]);
+fs.writeFileSync(path.resolve(STAGING_DIR, 'built.yml'), output.stdout.toString());

.circleci/config/jobs/lint.yml

@@ -0,0 +1,51 @@
+executor:
+  name: linux-docker
+  size: medium
+steps:
+  - checkout:
+      path: src/electron
+  - run:
+      name: Setup third_party Depot Tools
+      command: |
+        # "depot_tools" has to be checkout into "//third_party/depot_tools" so pylint.py can a "pylintrc" file.
+        git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git src/third_party/depot_tools
+        echo 'export PATH="$PATH:'"$PWD"'/src/third_party/depot_tools"' >> $BASH_ENV
+  - run:
+      name: Download GN Binary
+      command: |
+        chromium_revision="$(grep -A1 chromium_version src/electron/DEPS | tr -d '\n' | cut -d\' -f4)"
+        gn_version="$(curl -sL "https://chromium.googlesource.com/chromium/src/+/${chromium_revision}/DEPS?format=TEXT" | base64 -d | grep gn_version | head -n1 | cut -d\' -f4)"
+
+        cipd ensure -ensure-file - -root . \<<-CIPD
+        \$ServiceURL https://chrome-infra-packages.appspot.com/
+        @Subdir src/buildtools/linux64
+        gn/gn/linux-amd64 $gn_version
+        CIPD
+
+        echo 'export CHROMIUM_BUILDTOOLS_PATH="'"$PWD"'/src/buildtools"' >> $BASH_ENV
+  - run:
+      name: Download clang-format Binary
+      command: |
+        chromium_revision="$(grep -A1 chromium_version src/electron/DEPS | tr -d '\n' | cut -d\' -f4)"
+
+        sha1_path='buildtools/linux64/clang-format.sha1'
+        curl -sL "https://chromium.googlesource.com/chromium/src/+/${chromium_revision}/${sha1_path}?format=TEXT" | base64 -d > "src/${sha1_path}"
+
+        download_from_google_storage.py --no_resume --no_auth --bucket chromium-clang-format -s "src/${sha1_path}"
+  - run:
+      name: Run Lint
+      command: |
+        # gn.py tries to find a gclient root folder starting from the current dir.
+        # When it fails and returns "None" path, the whole script fails. Let's "fix" it.
+        touch .gclient
+        # Another option would be to checkout "buildtools" inside the Electron checkout,
+        # but then we would lint its contents (at least gn format), and it doesn't pass it.
+
+        cd src/electron
+        node script/yarn install --frozen-lockfile
+        node script/yarn lint
+  - run:
+      name: Run Script Typechecker
+      command: |
+        cd src/electron
+        node script/yarn tsc -p tsconfig.script.json

.circleci/config/package.json

@@ -0,0 +1,10 @@
+{
+  "name": "@electron/circleci-config",
+  "version": "0.0.0",
+  "private": true,
+  "license": "MIT",
+  "dependencies": {
+    "fs-extra": "^10.1.0",
+    "js-yaml": "^4.1.0"
+  }
+}

.circleci/config/yarn.lock

@@ -0,0 +1,43 @@
+# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
+# yarn lockfile v1
+
+
+argparse@^2.0.1:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/argparse/-/argparse-2.0.1.tgz#246f50f3ca78a3240f6c997e8a9bd1eac49e4b38"
+  integrity sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==
+
+fs-extra@^10.1.0:
+  version "10.1.0"
+  resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-10.1.0.tgz#02873cfbc4084dde127eaa5f9905eef2325d1abf"
+  integrity sha512-oRXApq54ETRj4eMiFzGnHWGy+zo5raudjuxN0b8H7s/RU2oW0Wvsx9O0ACRN/kRq9E8Vu/ReskGB5o3ji+FzHQ==
+  dependencies:
+    graceful-fs "^4.2.0"
+    jsonfile "^6.0.1"
+    universalify "^2.0.0"
+
+graceful-fs@^4.1.6, graceful-fs@^4.2.0:
+  version "4.2.10"
+  resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.10.tgz#147d3a006da4ca3ce14728c7aefc287c367d7a6c"
+  integrity sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA==
+
+js-yaml@^4.1.0:
+  version "4.1.0"
+  resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-4.1.0.tgz#c1fb65f8f5017901cdd2c951864ba18458a10602"
+  integrity sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==
+  dependencies:
+    argparse "^2.0.1"
+
+jsonfile@^6.0.1:
+  version "6.1.0"
+  resolved "https://registry.yarnpkg.com/jsonfile/-/jsonfile-6.1.0.tgz#bc55b2634793c679ec6403094eb13698a6ec0aae"
+  integrity sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==
+  dependencies:
+    universalify "^2.0.0"
+  optionalDependencies:
+    graceful-fs "^4.1.6"
+
+universalify@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/universalify/-/universalify-2.0.0.tgz#75a4984efedc4b08975c5aeb73f530d02df25717"
+  integrity sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ==

script/release/ci-release-build.js

@@ -110,12 +110,12 @@ async function getCircleCIWorkflowId (pipelineId) {
     switch (pipelineInfo.state) {
       case 'created': {
         const workflows = await circleCIRequest(`${pipelineInfoUrl}/workflow`, 'GET');
-        // The logic below expects two workflow.items: publish [0] & setup [1]
-        if (workflows.items.length === 2) {
+        // The logic below expects three workflow.items: publish, lint, & setup
+        if (workflows.items.length === 3) {
           workflowId = workflows.items.find(item => item.name.includes('publish')).id;
           break;
         }
-        console.log('Unxpected number of workflows, response was:', pipelineInfo);
+        console.log('Unxpected number of workflows, response was:', workflows);
         workflowId = -1;
         break;
       }