build: optimize the CI path where we update the patch files (#20854)

Currently the happy checkout takes 7 minutes and the sad checkout takes
30 minutes.  This updates our CI to run checkout twice for every job to
make the sad checkout take nearer 10 minutes instead.

.circleci/config.yml

@@ -192,6 +192,13 @@ step-restore-brew-cache: &step-restore-brew-cache
     keys:
       - v1-brew-cache-{{ arch }}
 
+step-save-brew-cache: &step-save-brew-cache
+  save_cache:
+    paths:
+      - /usr/local/Homebrew
+    key: v1-brew-cache-{{ arch }}
+    name: Persisting brew cache
+
 step-get-more-space-on-mac: &step-get-more-space-on-mac
   run:
     name: Free up space on MacOS
@@ -541,6 +548,93 @@ step-ninja-summary: &step-ninja-summary
     command: |
       python depot_tools/post_build_ninja_summary.py -C src/out/Default
 
+# Checkout Steps
+step-generate-deps-hash: &step-generate-deps-hash
+  run:
+    name: Generate DEPS Hash
+    command: node src/electron/script/generate-deps-hash.js
+
+step-touch-sync-done: &step-touch-sync-done
+  run:
+    name: Touch Sync Done
+    command: touch src/electron/.circle-sync-done
+
+# Restore exact src cache based on the hash of DEPS and patches/*
+# If no cache is matched EXACTLY then the .circle-sync-done file is empty
+# If a cache is matched EXACTLY then the .circle-sync-done file contains "done"
+step-maybe-restore-src-cache: &step-maybe-restore-src-cache
+  restore_cache:
+    paths:
+      - ./src
+    keys:
+      - v5-src-cache-{{ arch }}-{{ checksum "src/electron/.depshash" }}
+    name: Restoring src cache
+
+# Restore exact or closest git cache based on the hash of DEPS and .circle-sync-done
+# If the src cache was restored above then this will match an empty cache
+# If the src cache was not restored above then this will match a close git cache
+step-maybe-restore-git-cache: &step-maybe-restore-git-cache
+  restore_cache:
+    paths:
+      - ~/.gclient-cache
+    keys:
+      - v2-gclient-cache-{{ arch }}-{{ checksum "src/electron/.circle-sync-done" }}-{{ checksum "src/electron/DEPS" }}
+      - v2-gclient-cache-{{ arch }}-{{ checksum "src/electron/.circle-sync-done" }}
+    name: Conditionally restoring git cache
+
+step-set-git-cache-path: &step-set-git-cache-path
+  run:
+    name: Set GIT_CACHE_PATH to make gclient to use the cache
+    command: |
+      # CircleCI does not support interpolation when setting environment variables.
+      # https://circleci.com/docs/2.0/env-vars/#setting-an-environment-variable-in-a-shell-command
+      echo 'export GIT_CACHE_PATH="$HOME/.gclient-cache"' >> $BASH_ENV
+
+# Persist the git cache based on the hash of DEPS and .circle-sync-done
+# If the src cache was restored above then this will persist an empty cache
+step-save-git-cache: &step-save-git-cache
+  save_cache:
+    paths:
+      - ~/.gclient-cache
+    key: v2-gclient-cache-{{ arch }}-{{ checksum "src/electron/.circle-sync-done" }}-{{ checksum "src/electron/DEPS" }}
+    name: Persisting git cache
+
+step-run-electron-only-hooks: &step-run-electron-only-hooks
+  run:
+    name: Run Electron Only Hooks
+    command: gclient runhooks --spec="solutions=[{'name':'src/electron','url':None,'deps_file':'DEPS','custom_vars':{'process_deps':False},'managed':False}]"
+
+step-generate-deps-hash-cleanly: &step-generate-deps-hash-cleanly
+  run:
+    name: Generate DEPS Hash
+    command: (cd src/electron && git checkout .) && node src/electron/script/generate-deps-hash.js
+
+# Mark the sync as done for future cache saving
+step-mark-sync-done: &step-mark-sync-done
+  run:
+    name: Mark Sync Done
+    command: echo DONE > src/electron/.circle-sync-done
+
+# Minimize the size of the cache
+step-minimize-workspace-size-from-checkout: &step-minimize-workspace-size-from-checkout
+  run:
+    name: Remove some unused data to avoid storing it in the workspace/cache
+    command: |
+      rm -rf src/android_webview
+      rm -rf src/ios
+      rm -rf src/third_party/blink/web_tests
+      rm -rf src/third_party/blink/perf_tests
+      rm -rf src/third_party/hunspell_dictionaries
+      rm -rf src/third_party/WebKit/LayoutTests
+
+# Save the src cache based on the deps hash
+step-save-src-cache: &step-save-src-cache
+  save_cache:
+    paths:
+      - ./src
+    key: v5-src-cache-{{ arch }}-{{ checksum "src/electron/.depshash" }}
+    name: Persisting src cache
+
 # Lists of steps.
 steps-lint: &steps-lint
   steps:
@@ -586,7 +680,7 @@ steps-lint: &steps-lint
           node script/yarn install --frozen-lockfile
           node script/yarn lint
 
-steps-checkout: &steps-checkout
+steps-checkout-fast: &steps-checkout-fast
   steps:
     - *step-checkout-electron
     - *step-depot-tools-get
@@ -595,88 +689,57 @@ steps-checkout: &steps-checkout
     - *step-get-more-space-on-mac
     - *step-install-gnutar-on-mac
 
-    - run:
-        name: Generate DEPS Hash
-        command: node src/electron/script/generate-deps-hash.js
-    - run:
-        name: Touch Sync Done
-        command: touch src/electron/.circle-sync-done
-    # Restore exact src cache based on the hash of DEPS and patches/*
-    # If no cache is matched EXACTLY then the .circle-sync-done file is empty
-    # If a cache is matched EXACTLY then the .circle-sync-done file contains "done"
-    - restore_cache:
-        paths:
-          - ./src
-        keys:
-          - v5-src-cache-{{ arch }}-{{ checksum "src/electron/.depshash" }}
-        name: Restoring src cache
-    # Restore exact or closest git cache based on the hash of DEPS and .circle-sync-done
-    # If the src cache was restored above then this will match an empty cache
-    # If the src cache was not restored above then this will match a close git cache
-    - restore_cache:
-        paths:
-          - ~/.gclient-cache
-        keys:
-          - v2-gclient-cache-{{ arch }}-{{ checksum "src/electron/.circle-sync-done" }}-{{ checksum "src/electron/DEPS" }}
-          - v2-gclient-cache-{{ arch }}-{{ checksum "src/electron/.circle-sync-done" }}
-        name: Conditionally restoring git cache
-    - run:
-        name: Set GIT_CACHE_PATH to make gclient to use the cache
-        command: |
-          # CircleCI does not support interpolation when setting environment variables.
-          # https://circleci.com/docs/2.0/env-vars/#setting-an-environment-variable-in-a-shell-command
-          echo 'export GIT_CACHE_PATH="$HOME/.gclient-cache"' >> $BASH_ENV
+    - *step-generate-deps-hash
+    - *step-touch-sync-done
+    - *step-maybe-restore-src-cache
+    - *step-maybe-restore-git-cache
+    - *step-set-git-cache-path
     # This sync call only runs if .circle-sync-done is an EMPTY file
     - *step-gclient-sync
-    # Persist the git cache based on the hash of DEPS and .circle-sync-done
-    # If the src cache was restored above then this will persist an empty cache
-    - save_cache:
-        paths:
-          - ~/.gclient-cache
-        key: v2-gclient-cache-{{ arch }}-{{ checksum "src/electron/.circle-sync-done" }}-{{ checksum "src/electron/DEPS" }}
-        name: Persisting git cache
     # These next few steps reset Electron to the correct commit regardless of which cache was restored
     - run:
         name: Wipe Electron
         command: rm -rf src/electron
     - *step-checkout-electron
-    - run:
-        name: Run Electron Only Hooks
-        command: gclient runhooks --spec="solutions=[{'name':'src/electron','url':None,'deps_file':'DEPS','custom_vars':{'process_deps':False},'managed':False}]"
-    - run:
-        name: Generate DEPS Hash
-        command: (cd src/electron && git checkout .) && node src/electron/script/generate-deps-hash.js
-    # Mark the sync as done for future cache saving
-    - run:
-        name: Mark Sync Done
-        command: echo DONE > src/electron/.circle-sync-done
-    # Minimize the size of the cache
-    - run:
-        name: Remove some unused data to avoid storing it in the workspace/cache
-        command: |
-          rm -rf src/android_webview
-          rm -rf src/ios
-          rm -rf src/third_party/blink/web_tests
-          rm -rf src/third_party/blink/perf_tests
-          rm -rf src/third_party/hunspell_dictionaries
-          rm -rf src/third_party/WebKit/LayoutTests
-    # Save the src cache based on the deps hash
-    - save_cache:
-        paths:
-          - ./src
-        key: v5-src-cache-{{ arch }}-{{ checksum "src/electron/.depshash" }}
-        name: Persisting src cache
-    - save_cache:
-        paths:
-          - /usr/local/Homebrew
-        key: v1-brew-cache-{{ arch }}
-        name: Persisting brew cache
+    - *step-run-electron-only-hooks
+    - *step-generate-deps-hash-cleanly
+    - *step-mark-sync-done
+    - *step-minimize-workspace-size-from-checkout
     - persist_to_workspace:
         root: .
         paths:
           - depot_tools
           - src
 
+steps-checkout-and-save-cache: &steps-checkout-and-save-cache
+  steps:
+    - *step-checkout-electron
+    - *step-depot-tools-get
+    - *step-depot-tools-add-to-path
+    - *step-restore-brew-cache
+    - *step-get-more-space-on-mac
+    - *step-install-gnutar-on-mac
+
+    - *step-generate-deps-hash
+    - *step-touch-sync-done
+    - *step-maybe-restore-src-cache
+    - *step-maybe-restore-git-cache
+    - *step-set-git-cache-path
+    # This sync call only runs if .circle-sync-done is an EMPTY file
+    - *step-gclient-sync
+    - *step-save-git-cache
+    # These next few steps reset Electron to the correct commit regardless of which cache was restored
+    - run:
+        name: Wipe Electron
+        command: rm -rf src/electron
+    - *step-checkout-electron
+    - *step-run-electron-only-hooks
+    - *step-generate-deps-hash-cleanly
+    - *step-mark-sync-done
+    - *step-minimize-workspace-size-from-checkout
+    - *step-save-src-cache
+    - *step-save-brew-cache
+
 steps-electron-gn-check: &steps-electron-gn-check
   steps:
     - attach_workspace:
@@ -955,33 +1018,47 @@ jobs:
     <<: *steps-lint
 
   # Layer 1: Checkout.
-  linux-checkout:
+  linux-checkout-fast:
     <<: *machine-linux-2xlarge
     environment:
       <<: *env-linux-2xlarge
       GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True'
-    <<: *steps-checkout
+    <<: *steps-checkout-fast
+
+  linux-checkout-and-save-cache:
+    <<: *machine-linux-2xlarge
+    environment:
+      <<: *env-linux-2xlarge
+      GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True'
+    <<: *steps-checkout-and-save-cache
 
   linux-checkout-for-native-tests:
     <<: *machine-linux-2xlarge
     environment:
       <<: *env-linux-2xlarge
       GCLIENT_EXTRA_ARGS: '--custom-var=checkout_pyyaml=True'
-    <<: *steps-checkout
+    <<: *steps-checkout-fast
 
   linux-checkout-for-native-tests-with-no-patches:
     <<: *machine-linux-2xlarge
     environment:
       <<: *env-linux-2xlarge
       GCLIENT_EXTRA_ARGS: '--custom-var=apply_patches=False --custom-var=checkout_pyyaml=True'
-    <<: *steps-checkout
+    <<: *steps-checkout-fast
 
-  mac-checkout:
+  mac-checkout-fast:
     <<: *machine-linux-2xlarge
     environment:
       <<: *env-linux-2xlarge
       GCLIENT_EXTRA_ARGS: '--custom-var=checkout_mac=True --custom-var=host_os=mac'
-    <<: *steps-checkout
+    <<: *steps-checkout-fast
+
+  mac-checkout-and-save-cache:
+    <<: *machine-linux-2xlarge
+    environment:
+      <<: *env-linux-2xlarge
+      GCLIENT_EXTRA_ARGS: '--custom-var=checkout_mac=True --custom-var=host_os=mac'
+    <<: *steps-checkout-and-save-cache
 
   # Layer 2: Builds.
   linux-x64-debug:
@@ -1602,23 +1679,24 @@ workflows:
 
   build-linux:
     jobs:
-      - linux-checkout
+      - linux-checkout-fast
+      - linux-checkout-and-save-cache
 
       - linux-x64-debug:
           requires:
-            - linux-checkout
+            - linux-checkout-fast
       - linux-x64-debug-gn-check:
           requires:
-            - linux-checkout
+            - linux-checkout-fast
       - linux-x64-testing:
           requires:
-            - linux-checkout
+            - linux-checkout-fast
       - linux-x64-testing-no-run-as-node:
           requires:
-            - linux-checkout
+            - linux-checkout-fast
       - linux-x64-testing-gn-check:
           requires:
-            - linux-checkout
+            - linux-checkout-fast
       - linux-x64-testing-tests:
           requires:
             - linux-x64-testing
@@ -1631,10 +1709,10 @@ workflows:
 
       - linux-ia32-debug:
           requires:
-            - linux-checkout
+            - linux-checkout-fast
       - linux-ia32-testing:
           requires:
-            - linux-checkout
+            - linux-checkout-fast
       - linux-ia32-testing-tests:
           requires:
             - linux-ia32-testing
@@ -1647,42 +1725,44 @@ workflows:
 
       - linux-arm-debug:
           requires:
-            - linux-checkout
+            - linux-checkout-fast
       - linux-arm-testing:
           requires:
-            - linux-checkout
+            - linux-checkout-fast
 
       - linux-arm64-debug:
           requires:
-            - linux-checkout
+            - linux-checkout-fast
       - linux-arm64-debug-gn-check:
           requires:
-            - linux-checkout
+            - linux-checkout-fast
       - linux-arm64-testing:
           requires:
-            - linux-checkout
+            - linux-checkout-fast
       - linux-arm64-testing-gn-check:
           requires:
-            - linux-checkout
+            - linux-checkout-fast
 
   build-mac:
     jobs:
-      - mac-checkout
+      - mac-checkout-fast
+      - mac-checkout-and-save-cache
+
       - osx-testing:
           requires:
-            - mac-checkout
+            - mac-checkout-fast
 
       - osx-debug:
           requires:
-            - mac-checkout
+            - mac-checkout-fast
 
       - osx-debug-gn-check:
           requires:
-            - mac-checkout
+            - mac-checkout-fast
 
       - osx-testing-gn-check:
           requires:
-            - mac-checkout
+            - mac-checkout-fast
 
       - osx-testing-tests:
           requires:
@@ -1690,19 +1770,19 @@ workflows:
 
       - mas-testing:
           requires:
-            - mac-checkout
+            - mac-checkout-fast
 
       - mas-debug:
           requires:
-            - mac-checkout
+            - mac-checkout-fast
 
       - mas-debug-gn-check:
           requires:
-            - mac-checkout
+            - mac-checkout-fast
 
       - mas-testing-gn-check:
           requires:
-            - mac-checkout
+            - mac-checkout-fast
 
       - mas-testing-tests:
           requires: