Browse Source

build: optimize the CI path where we update the patch files (#19851)

Currently the happy checkout takes 7 minutes and the sad checkout takes
30 minutes.  This updates our CI to run checkout twice for every job to
make the sad checkout take nearer 10 minutes instead.
Samuel Attard 5 years ago
parent
commit
612e3f7ff7
1 changed files with 179 additions and 99 deletions
  1. 179 99
      .circleci/config.yml

+ 179 - 99
.circleci/config.yml

@@ -186,6 +186,13 @@ step-restore-brew-cache: &step-restore-brew-cache
     keys:
       - v1-brew-cache-{{ arch }}
 
+step-save-brew-cache: &step-save-brew-cache
+  save_cache:
+    paths:
+      - /usr/local/Homebrew
+    key: v1-brew-cache-{{ arch }}
+    name: Persisting brew cache
+
 step-get-more-space-on-mac: &step-get-more-space-on-mac
   run:
     name: Free up space on MacOS
@@ -535,6 +542,93 @@ step-ninja-summary: &step-ninja-summary
     command: |
       python depot_tools/post_build_ninja_summary.py -C src/out/Default
 
+# Checkout Steps
+step-generate-deps-hash: &step-generate-deps-hash
+  run:
+    name: Generate DEPS Hash
+    command: node src/electron/script/generate-deps-hash.js
+
+step-touch-sync-done: &step-touch-sync-done
+  run:
+    name: Touch Sync Done
+    command: touch src/electron/.circle-sync-done
+
+# Restore exact src cache based on the hash of DEPS and patches/*
+# If no cache is matched EXACTLY then the .circle-sync-done file is empty
+# If a cache is matched EXACTLY then the .circle-sync-done file contains "done"
+step-maybe-restore-src-cache: &step-maybe-restore-src-cache
+  restore_cache:
+    paths:
+      - ./src
+    keys:
+      - v5-src-cache-{{ arch }}-{{ checksum "src/electron/.depshash" }}
+    name: Restoring src cache
+
+# Restore exact or closest git cache based on the hash of DEPS and .circle-sync-done
+# If the src cache was restored above then this will match an empty cache
+# If the src cache was not restored above then this will match a close git cache
+step-maybe-restore-git-cache: &step-maybe-restore-git-cache
+  restore_cache:
+    paths:
+      - ~/.gclient-cache
+    keys:
+      - v2-gclient-cache-{{ arch }}-{{ checksum "src/electron/.circle-sync-done" }}-{{ checksum "src/electron/DEPS" }}
+      - v2-gclient-cache-{{ arch }}-{{ checksum "src/electron/.circle-sync-done" }}
+    name: Conditionally restoring git cache
+
+step-set-git-cache-path: &step-set-git-cache-path
+  run:
+    name: Set GIT_CACHE_PATH to make gclient to use the cache
+    command: |
+      # CircleCI does not support interpolation when setting environment variables.
+      # https://circleci.com/docs/2.0/env-vars/#setting-an-environment-variable-in-a-shell-command
+      echo 'export GIT_CACHE_PATH="$HOME/.gclient-cache"' >> $BASH_ENV
+
+# Persist the git cache based on the hash of DEPS and .circle-sync-done
+# If the src cache was restored above then this will persist an empty cache
+step-save-git-cache: &step-save-git-cache
+  save_cache:
+    paths:
+      - ~/.gclient-cache
+    key: v2-gclient-cache-{{ arch }}-{{ checksum "src/electron/.circle-sync-done" }}-{{ checksum "src/electron/DEPS" }}
+    name: Persisting git cache
+
+step-run-electron-only-hooks: &step-run-electron-only-hooks
+  run:
+    name: Run Electron Only Hooks
+    command: gclient runhooks --spec="solutions=[{'name':'src/electron','url':None,'deps_file':'DEPS','custom_vars':{'process_deps':False},'managed':False}]"
+
+step-generate-deps-hash-cleanly: &step-generate-deps-hash-cleanly
+  run:
+    name: Generate DEPS Hash
+    command: (cd src/electron && git checkout .) && node src/electron/script/generate-deps-hash.js
+
+# Mark the sync as done for future cache saving
+step-mark-sync-done: &step-mark-sync-done
+  run:
+    name: Mark Sync Done
+    command: echo DONE > src/electron/.circle-sync-done
+
+# Minimize the size of the cache
+step-minimize-workspace-size-from-checkout: &step-minimize-workspace-size-from-checkout
+  run:
+    name: Remove some unused data to avoid storing it in the workspace/cache
+    command: |
+      rm -rf src/android_webview
+      rm -rf src/ios
+      rm -rf src/third_party/blink/web_tests
+      rm -rf src/third_party/blink/perf_tests
+      rm -rf src/third_party/hunspell_dictionaries
+      rm -rf src/third_party/WebKit/LayoutTests
+
+# Save the src cache based on the deps hash
+step-save-src-cache: &step-save-src-cache
+  save_cache:
+    paths:
+      - ./src
+    key: v5-src-cache-{{ arch }}-{{ checksum "src/electron/.depshash" }}
+    name: Persisting src cache
+
 # Lists of steps.
 steps-lint: &steps-lint
   steps:
@@ -580,7 +674,7 @@ steps-lint: &steps-lint
           node script/yarn install --frozen-lockfile
           node script/yarn lint
 
-steps-checkout: &steps-checkout
+steps-checkout-fast: &steps-checkout-fast
   steps:
     - *step-checkout-electron
     - *step-depot-tools-get
@@ -589,88 +683,57 @@ steps-checkout: &steps-checkout
     - *step-get-more-space-on-mac
     - *step-install-gnutar-on-mac
 
-    - run:
-        name: Generate DEPS Hash
-        command: node src/electron/script/generate-deps-hash.js
-    - run:
-        name: Touch Sync Done
-        command: touch src/electron/.circle-sync-done
-    # Restore exact src cache based on the hash of DEPS and patches/*
-    # If no cache is matched EXACTLY then the .circle-sync-done file is empty
-    # If a cache is matched EXACTLY then the .circle-sync-done file contains "done"
-    - restore_cache:
-        paths:
-          - ./src
-        keys:
-          - v5-src-cache-{{ arch }}-{{ checksum "src/electron/.depshash" }}
-        name: Restoring src cache
-    # Restore exact or closest git cache based on the hash of DEPS and .circle-sync-done
-    # If the src cache was restored above then this will match an empty cache
-    # If the src cache was not restored above then this will match a close git cache
-    - restore_cache:
-        paths:
-          - ~/.gclient-cache
-        keys:
-          - v2-gclient-cache-{{ arch }}-{{ checksum "src/electron/.circle-sync-done" }}-{{ checksum "src/electron/DEPS" }}
-          - v2-gclient-cache-{{ arch }}-{{ checksum "src/electron/.circle-sync-done" }}
-        name: Conditionally restoring git cache
-    - run:
-        name: Set GIT_CACHE_PATH to make gclient to use the cache
-        command: |
-          # CircleCI does not support interpolation when setting environment variables.
-          # https://circleci.com/docs/2.0/env-vars/#setting-an-environment-variable-in-a-shell-command
-          echo 'export GIT_CACHE_PATH="$HOME/.gclient-cache"' >> $BASH_ENV
+    - *step-generate-deps-hash
+    - *step-touch-sync-done
+    - *step-maybe-restore-src-cache
+    - *step-maybe-restore-git-cache
+    - *step-set-git-cache-path
     # This sync call only runs if .circle-sync-done is an EMPTY file
     - *step-gclient-sync
-    # Persist the git cache based on the hash of DEPS and .circle-sync-done
-    # If the src cache was restored above then this will persist an empty cache
-    - save_cache:
-        paths:
-          - ~/.gclient-cache
-        key: v2-gclient-cache-{{ arch }}-{{ checksum "src/electron/.circle-sync-done" }}-{{ checksum "src/electron/DEPS" }}
-        name: Persisting git cache
     # These next few steps reset Electron to the correct commit regardless of which cache was restored
     - run:
         name: Wipe Electron
         command: rm -rf src/electron
     - *step-checkout-electron
-    - run:
-        name: Run Electron Only Hooks
-        command: gclient runhooks --spec="solutions=[{'name':'src/electron','url':None,'deps_file':'DEPS','custom_vars':{'process_deps':False},'managed':False}]"
-    - run:
-        name: Generate DEPS Hash
-        command: (cd src/electron && git checkout .) && node src/electron/script/generate-deps-hash.js
-    # Mark the sync as done for future cache saving
-    - run:
-        name: Mark Sync Done
-        command: echo DONE > src/electron/.circle-sync-done
-    # Minimize the size of the cache
-    - run:
-        name: Remove some unused data to avoid storing it in the workspace/cache
-        command: |
-          rm -rf src/android_webview
-          rm -rf src/ios
-          rm -rf src/third_party/blink/web_tests
-          rm -rf src/third_party/blink/perf_tests
-          rm -rf src/third_party/hunspell_dictionaries
-          rm -rf src/third_party/WebKit/LayoutTests
-    # Save the src cache based on the deps hash
-    - save_cache:
-        paths:
-          - ./src
-        key: v5-src-cache-{{ arch }}-{{ checksum "src/electron/.depshash" }}
-        name: Persisting src cache
-    - save_cache:
-        paths:
-          - /usr/local/Homebrew
-        key: v1-brew-cache-{{ arch }}
-        name: Persisting brew cache
+    - *step-run-electron-only-hooks
+    - *step-generate-deps-hash-cleanly
+    - *step-mark-sync-done
+    - *step-minimize-workspace-size-from-checkout
     - persist_to_workspace:
         root: .
         paths:
           - depot_tools
           - src
 
+steps-checkout-and-save-cache: &steps-checkout-and-save-cache
+  steps:
+    - *step-checkout-electron
+    - *step-depot-tools-get
+    - *step-depot-tools-add-to-path
+    - *step-restore-brew-cache
+    - *step-get-more-space-on-mac
+    - *step-install-gnutar-on-mac
+
+    - *step-generate-deps-hash
+    - *step-touch-sync-done
+    - *step-maybe-restore-src-cache
+    - *step-maybe-restore-git-cache
+    - *step-set-git-cache-path
+    # This sync call only runs if .circle-sync-done is an EMPTY file
+    - *step-gclient-sync
+    - *step-save-git-cache
+    # These next few steps reset Electron to the correct commit regardless of which cache was restored
+    - run:
+        name: Wipe Electron
+        command: rm -rf src/electron
+    - *step-checkout-electron
+    - *step-run-electron-only-hooks
+    - *step-generate-deps-hash-cleanly
+    - *step-mark-sync-done
+    - *step-minimize-workspace-size-from-checkout
+    - *step-save-src-cache
+    - *step-save-brew-cache
+
 steps-electron-gn-check: &steps-electron-gn-check
   steps:
     - attach_workspace:
@@ -949,33 +1012,47 @@ jobs:
     <<: *steps-lint
 
   # Layer 1: Checkout.
-  linux-checkout:
+  linux-checkout-fast:
     <<: *machine-linux-2xlarge
     environment:
       <<: *env-linux-2xlarge
       GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True'
-    <<: *steps-checkout
+    <<: *steps-checkout-fast
+
+  linux-checkout-and-save-cache:
+    <<: *machine-linux-2xlarge
+    environment:
+      <<: *env-linux-2xlarge
+      GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True'
+    <<: *steps-checkout-and-save-cache
 
   linux-checkout-for-native-tests:
     <<: *machine-linux-2xlarge
     environment:
       <<: *env-linux-2xlarge
       GCLIENT_EXTRA_ARGS: '--custom-var=checkout_pyyaml=True'
-    <<: *steps-checkout
+    <<: *steps-checkout-fast
 
   linux-checkout-for-native-tests-with-no-patches:
     <<: *machine-linux-2xlarge
     environment:
       <<: *env-linux-2xlarge
       GCLIENT_EXTRA_ARGS: '--custom-var=apply_patches=False --custom-var=checkout_pyyaml=True'
-    <<: *steps-checkout
+    <<: *steps-checkout-fast
 
-  mac-checkout:
+  mac-checkout-fast:
     <<: *machine-linux-2xlarge
     environment:
       <<: *env-linux-2xlarge
       GCLIENT_EXTRA_ARGS: '--custom-var=checkout_mac=True --custom-var=host_os=mac'
-    <<: *steps-checkout
+    <<: *steps-checkout-fast
+
+  mac-checkout-and-save-cache:
+    <<: *machine-linux-2xlarge
+    environment:
+      <<: *env-linux-2xlarge
+      GCLIENT_EXTRA_ARGS: '--custom-var=checkout_mac=True --custom-var=host_os=mac'
+    <<: *steps-checkout-and-save-cache
 
   # Layer 2: Builds.
   linux-x64-debug:
@@ -1596,23 +1673,24 @@ workflows:
 
   build-linux:
     jobs:
-      - linux-checkout
+      - linux-checkout-fast
+      - linux-checkout-and-save-cache
 
       - linux-x64-debug:
           requires:
-            - linux-checkout
+            - linux-checkout-fast
       - linux-x64-debug-gn-check:
           requires:
-            - linux-checkout
+            - linux-checkout-fast
       - linux-x64-testing:
           requires:
-            - linux-checkout
+            - linux-checkout-fast
       - linux-x64-testing-no-run-as-node:
           requires:
-            - linux-checkout
+            - linux-checkout-fast
       - linux-x64-testing-gn-check:
           requires:
-            - linux-checkout
+            - linux-checkout-fast
       - linux-x64-testing-tests:
           requires:
             - linux-x64-testing
@@ -1625,10 +1703,10 @@ workflows:
 
       - linux-ia32-debug:
           requires:
-            - linux-checkout
+            - linux-checkout-fast
       - linux-ia32-testing:
           requires:
-            - linux-checkout
+            - linux-checkout-fast
       - linux-ia32-testing-tests:
           requires:
             - linux-ia32-testing
@@ -1641,42 +1719,44 @@ workflows:
 
       - linux-arm-debug:
           requires:
-            - linux-checkout
+            - linux-checkout-fast
       - linux-arm-testing:
           requires:
-            - linux-checkout
+            - linux-checkout-fast
 
       - linux-arm64-debug:
           requires:
-            - linux-checkout
+            - linux-checkout-fast
       - linux-arm64-debug-gn-check:
           requires:
-            - linux-checkout
+            - linux-checkout-fast
       - linux-arm64-testing:
           requires:
-            - linux-checkout
+            - linux-checkout-fast
       - linux-arm64-testing-gn-check:
           requires:
-            - linux-checkout
+            - linux-checkout-fast
 
   build-mac:
     jobs:
-      - mac-checkout
+      - mac-checkout-fast
+      - mac-checkout-and-save-cache
+
       - osx-testing:
           requires:
-            - mac-checkout
+            - mac-checkout-fast
 
       - osx-debug:
           requires:
-            - mac-checkout
+            - mac-checkout-fast
 
       - osx-debug-gn-check:
           requires:
-            - mac-checkout
+            - mac-checkout-fast
 
       - osx-testing-gn-check:
           requires:
-            - mac-checkout
+            - mac-checkout-fast
 
       - osx-testing-tests:
           requires:
@@ -1684,19 +1764,19 @@ workflows:
 
       - mas-testing:
           requires:
-            - mac-checkout
+            - mac-checkout-fast
 
       - mas-debug:
           requires:
-            - mac-checkout
+            - mac-checkout-fast
 
       - mas-debug-gn-check:
           requires:
-            - mac-checkout
+            - mac-checkout-fast
 
       - mas-testing-gn-check:
           requires:
-            - mac-checkout
+            - mac-checkout-fast
 
       - mas-testing-tests:
           requires: