build: improve CI speeds and reduce CI costs (#34184)

build: improve CI speeds and reduce CI costs (#33904) (#33953)

* build: improve CI speeds and reduce CI costs (#33904)

* remove third_party/electron_node:overlapped-checker

target isn't present in older versions

* build: use original arch logic

Co-authored-by: John Kleinschmidt <[email protected]>
Co-authored-by: Samuel Attard <[email protected]>

Co-authored-by: Keeley Hammond <[email protected]>
Co-authored-by: John Kleinschmidt <[email protected]>

.circleci/config/base.yml

@@ -330,6 +330,19 @@ step-setup-goma-for-build: &step-setup-goma-for-build
       echo 'export LOCAL_GOMA_DIR='`node -e "console.log(require('./src/utils/goma.js').dir)"` >> $BASH_ENV
       echo 'export GOMA_FALLBACK_ON_AUTH_FAILURE=true' >> $BASH_ENV
       cd ..
+      touch "${TMPDIR:=/tmp}"/.goma-ready
+    background: true
+
+step-wait-for-goma: &step-wait-for-goma
+  run:
+    name: Wait for Goma
+    command: |
+      until [ -f "${TMPDIR:=/tmp}"/.goma-ready ]
+      do
+          sleep 5
+      done
+      echo "Goma ready"
+    no_output_timeout: 2m
 
 step-restore-brew-cache: &step-restore-brew-cache
   restore_cache:
@@ -543,14 +556,6 @@ step-electron-build: &step-electron-build
       cp out/Default/.ninja_log out/electron_ninja_log
       node electron/script/check-symlinks.js
 
-step-native-unittests-build: &step-native-unittests-build
-  run:
-    name: Build native test targets
-    no_output_timeout: 30m
-    command: |
-      cd src
-      ninja -C out/Default shell_browser_ui_unittests -j $NUMBER_OF_NINJA_PROCESSES
-
 step-maybe-electron-dist-strip: &step-maybe-electron-dist-strip
   run:
     name: Strip electron binaries
@@ -569,40 +574,6 @@ step-maybe-electron-dist-strip: &step-maybe-electron-dist-strip
         electron/script/add-debug-link.py --target-cpu="$target_cpu" --debug-dir=out/Default/debug
       fi
 
-step-electron-dist-build: &step-electron-dist-build
-  run:
-    name: Build dist.zip
-    command: |
-      cd src
-      if [ "$SKIP_DIST_ZIP" != "1" ]; then
-        ninja -C out/Default electron:electron_dist_zip
-        if [ "$CHECK_DIST_MANIFEST" == "1" ]; then
-          if [ "`uname`" == "Darwin" ]; then
-            target_os=mac
-            target_cpu=x64
-            if [ x"$MAS_BUILD" == x"true" ]; then
-              target_os=mac_mas
-            fi
-            if [ "$TARGET_ARCH" == "arm64" ]; then
-              target_cpu=arm64
-            fi
-          elif [ "`uname`" == "Linux" ]; then
-            target_os=linux
-            if [ x"$TARGET_ARCH" == x ]; then
-              target_cpu=x64
-            elif [ "$TARGET_ARCH" == "ia32" ]; then
-              target_cpu=x86
-            else
-              target_cpu="$TARGET_ARCH"
-            fi
-          else
-            echo "Unknown system: `uname`"
-            exit 1
-          fi
-          electron/script/zip_manifests/check-zip-manifest.py out/Default/dist.zip electron/script/zip_manifests/dist_zip.$target_os.$target_cpu.manifest
-        fi
-      fi
-
 step-electron-chromedriver-build: &step-electron-chromedriver-build
   run:
     name: Build chromedriver.zip
@@ -655,7 +626,6 @@ step-persist-data-for-tests: &step-persist-data-for-tests
       - src/out/Default/dist.zip
       - src/out/Default/mksnapshot.zip
       - src/out/Default/chromedriver.zip
-      - src/out/Default/shell_browser_ui_unittests
       - src/out/Default/gen/node_headers
       - src/out/ffmpeg/ffmpeg.zip
       - src/electron
@@ -766,6 +736,7 @@ step-show-goma-stats: &step-show-goma-stats
       $LOCAL_GOMA_DIR/diagnose_goma_log.py
       true
     when: always
+    background: true
 
 step-mksnapshot-build: &step-mksnapshot-build
   run:
@@ -1008,11 +979,15 @@ step-ts-compile: &step-ts-compile
 # List of all steps.
 steps-electron-gn-check: &steps-electron-gn-check
   steps:
-    - attach_workspace:
-        at: .
+    - *step-checkout-electron
+    - *step-depot-tools-get
     - *step-depot-tools-add-to-path
     - *step-setup-env-for-build
     - *step-setup-goma-for-build
+    - *step-generate-deps-hash
+    - *step-touch-sync-done
+    - maybe-restore-portaled-src-cache
+    - *step-wait-for-goma
     - *step-gn-gen-default
     - *step-gn-check
 
@@ -1025,6 +1000,7 @@ steps-electron-ts-compile-for-doc-change: &steps-electron-ts-compile-for-doc-cha
     - *step-restore-brew-cache
     - *step-install-gnutar-on-mac
     - *step-get-more-space-on-mac
+    - *step-setup-goma-for-build
     - *step-generate-deps-hash
     - *step-touch-sync-done
     - maybe-restore-portaled-src-cache
@@ -1044,7 +1020,7 @@ steps-electron-ts-compile-for-doc-change: &steps-electron-ts-compile-for-doc-cha
 
     - *step-depot-tools-add-to-path
     - *step-setup-env-for-build
-    - *step-setup-goma-for-build
+    - *step-wait-for-goma
     - *step-get-more-space-on-mac
     - *step-install-npm-deps-on-mac
     - *step-fix-sync-on-mac
@@ -1060,6 +1036,7 @@ steps-native-tests: &steps-native-tests
     - *step-depot-tools-add-to-path
     - *step-setup-env-for-build
     - *step-setup-goma-for-build
+    - *step-wait-for-goma
     - *step-gn-gen-default
 
     - run:
@@ -1245,7 +1222,6 @@ commands:
               fi
             }
             mv_if_exist src/out/Default/dist.zip
-            mv_if_exist src/out/Default/shell_browser_ui_unittests
             mv_if_exist src/out/Default/gen/node_headers.tar.gz
             mv_if_exist src/out/Default/symbols.zip
             mv_if_exist src/out/Default/mksnapshot.zip
@@ -1282,6 +1258,46 @@ commands:
       - *step-checkout-electron
       - *step-run-electron-only-hooks
       - *step-generate-deps-hash-cleanly
+
+  step-electron-dist-build:
+    parameters:
+      additional-targets:
+        type: string
+        default: ''
+    steps:
+      - run:
+          name: Build dist.zip
+          command: |
+            cd src
+            if [ "$SKIP_DIST_ZIP" != "1" ]; then
+              ninja -C out/Default electron:electron_dist_zip << parameters.additional-targets >>
+              if [ "$CHECK_DIST_MANIFEST" == "1" ]; then
+                if [ "`uname`" == "Darwin" ]; then
+                  target_os=mac
+                  target_cpu=x64
+                  if [ x"$MAS_BUILD" == x"true" ]; then
+                    target_os=mac_mas
+                  fi
+                  if [ "$TARGET_ARCH" == "arm64" ]; then
+                    target_cpu=arm64
+                  fi
+                elif [ "`uname`" == "Linux" ]; then
+                  target_os=linux
+                  if [ x"$TARGET_ARCH" == x ]; then
+                    target_cpu=x64
+                  elif [ "$TARGET_ARCH" == "ia32" ]; then
+                    target_cpu=x86
+                  else
+                    target_cpu="$TARGET_ARCH"
+                  fi
+                else
+                  echo "Unknown system: `uname`"
+                  exit 1
+                fi
+                electron/script/zip_manifests/check-zip-manifest.py out/Default/dist.zip electron/script/zip_manifests/dist_zip.$target_os.$target_cpu.manifest
+              fi
+            fi
+
   electron-build:
     parameters:
       attach:
@@ -1327,6 +1343,10 @@ commands:
       - *step-restore-brew-cache
       - *step-install-gnutar-on-mac
       - *step-save-brew-cache
+      - when:
+          condition: << parameters.build >>
+          steps:
+            - *step-setup-goma-for-build
       - when:
           condition: << parameters.checkout-and-assume-cache >>
           steps:
@@ -1426,7 +1446,7 @@ commands:
           steps:
             - *step-depot-tools-add-to-path
             - *step-setup-env-for-build
-            - *step-setup-goma-for-build
+            - *step-wait-for-goma
             - *step-get-more-space-on-mac
             - *step-fix-sync-on-mac
             - *step-delete-git-directories
@@ -1439,13 +1459,8 @@ commands:
             - *step-gn-gen-default
             - *step-electron-build
             - *step-maybe-electron-dist-strip
-            - *step-electron-dist-build
-
-            # Native test targets
-            - *step-native-unittests-build
-
-            # Node.js headers
-            - *step-nodejs-headers-build
+            - step-electron-dist-build:
+                additional-targets: shell_browser_ui_unittests third_party/electron_node:headers electron:hunspell_dictionaries_zip
 
             - *step-show-goma-stats
 
@@ -1463,13 +1478,14 @@ commands:
                   - *step-ffmpeg-gn-gen
                   - *step-ffmpeg-build
 
-            # hunspell
-            - *step-hunspell-build
-
       # Save all data needed for a further tests run.
       - when:
           condition: << parameters.persist >>
           steps:
+            - *step-minimize-workspace-size-from-checkout
+            - run: |
+                rm -rf src/third_party/electron_node/deps/openssl
+                rm -rf src/third_party/electron_node/deps/v8
             - *step-persist-data-for-tests
 
       - when:
@@ -1535,6 +1551,7 @@ commands:
       - *step-fix-sync-on-mac
       - *step-setup-env-for-build
       - *step-setup-goma-for-build
+      - *step-wait-for-goma
       - *step-gn-gen-default
 
       # Electron app
@@ -1542,7 +1559,7 @@ commands:
       - *step-show-goma-stats
       - *step-maybe-generate-breakpad-symbols
       - *step-maybe-electron-dist-strip
-      - *step-electron-dist-build
+      - step-electron-dist-build
       - *step-maybe-zip-symbols
 
       # mksnapshot
@@ -1585,20 +1602,10 @@ jobs:
     <<: *steps-electron-ts-compile-for-doc-change
 
   # Layer 1: Checkout.
-  linux-checkout-for-workspace:
-    executor: linux-docker
-    environment:
-      <<: *env-linux-2xlarge
-      GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True'
-    steps:
-      - electron-build:
-          persist: false
-          build: false
-          checkout: true
-          persist-checkout: true
-
   linux-make-src-cache:
-    executor: linux-docker
+    executor:
+      name: linux-docker
+      size: xlarge
     environment:
       <<: *env-linux-2xlarge
       GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True'
@@ -1649,22 +1656,10 @@ jobs:
           persist-checkout: true
           restore-src-cache: false
 
-  mac-checkout-for-workspace:
-    executor: linux-docker
-    environment:
-      <<: *env-linux-2xlarge
-      <<: *env-testing-build
-      <<: *env-macos-build
-      GCLIENT_EXTRA_ARGS: '--custom-var=checkout_mac=True --custom-var=host_os=mac'
-    steps:
-      - electron-build:
-          persist: false
-          build: false
-          checkout: true
-          persist-checkout: true
-
   mac-make-src-cache:
-    executor: linux-docker
+    executor:
+      name: linux-docker
+      size: xlarge
     environment:
       <<: *env-linux-2xlarge
       <<: *env-testing-build
@@ -1689,7 +1684,8 @@ jobs:
     steps:
       - electron-build:
           persist: true
-          checkout: true
+          checkout: false
+          checkout-and-assume-cache: true
           use-out-cache: false
 
   linux-x64-testing-asan:
@@ -1729,6 +1725,7 @@ jobs:
     environment:
       <<: *env-linux-medium
       <<: *env-testing-build
+      GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True'
     <<: *steps-electron-gn-check
 
   linux-x64-release:
@@ -1825,7 +1822,8 @@ jobs:
     steps:
       - electron-build:
           persist: true
-          checkout: true
+          checkout: false
+          checkout-and-assume-cache: true
           use-out-cache: false
 
   linux-arm-release:
@@ -1877,7 +1875,8 @@ jobs:
     steps:
       - electron-build:
           persist: true
-          checkout: true
+          checkout: false
+          checkout-and-assume-cache: true
           use-out-cache: false
 
   linux-arm64-testing-gn-check:
@@ -1888,6 +1887,7 @@ jobs:
       <<: *env-linux-medium
       <<: *env-arm64
       <<: *env-testing-build
+      GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True'
     <<: *steps-electron-gn-check
 
   linux-arm64-release:
@@ -1947,6 +1947,7 @@ jobs:
     environment:
       <<: *env-machine-mac
       <<: *env-testing-build
+      GCLIENT_EXTRA_ARGS: '--custom-var=checkout_mac=True --custom-var=host_os=mac'
     <<: *steps-electron-gn-check
 
   osx-publish-x64-skip-checkout:
@@ -2029,6 +2030,7 @@ jobs:
       <<: *env-machine-mac
       <<: *env-mas
       <<: *env-testing-build
+      GCLIENT_EXTRA_ARGS: '--custom-var=checkout_mac=True --custom-var=host_os=mac'
     <<: *steps-electron-gn-check
 
   mas-publish-x64-skip-checkout:
@@ -2355,14 +2357,19 @@ workflows:
         - equal: [false, << pipeline.parameters.run-linux-publish >>]
         - equal: [true, << pipeline.parameters.run-build-linux >>]
     jobs:
-      - linux-checkout-for-workspace
       - linux-make-src-cache
-      - linux-x64-testing
-      - linux-x64-testing-asan
-      - linux-x64-testing-no-run-as-node
+      - linux-x64-testing:
+          requires:
+            - linux-make-src-cache
+      - linux-x64-testing-asan:
+          requires:
+            - linux-make-src-cache
+      - linux-x64-testing-no-run-as-node:
+          requires:
+            - linux-make-src-cache
       - linux-x64-testing-gn-check:
           requires:
-            - linux-checkout-for-workspace
+            - linux-make-src-cache
       - linux-x64-testing-tests:
           requires:
             - linux-x64-testing
@@ -2375,7 +2382,9 @@ workflows:
       - linux-x64-testing-node:
           requires:
             - linux-x64-testing
-      - linux-ia32-testing
+      - linux-ia32-testing:
+          requires:
+            - linux-make-src-cache
       - linux-ia32-testing-tests:
           requires:
             - linux-ia32-testing
@@ -2385,7 +2394,9 @@ workflows:
       - linux-ia32-testing-node:
           requires:
             - linux-ia32-testing
-      - linux-arm-testing
+      - linux-arm-testing:
+          requires:
+            - linux-make-src-cache
       - linux-arm-testing-tests:
           filters:
             branches:
@@ -2393,7 +2404,9 @@ workflows:
               ignore: /pull\/[0-9]+/        
           requires:
             - linux-arm-testing      
-      - linux-arm64-testing
+      - linux-arm64-testing:
+          requires:
+            - linux-make-src-cache
       - linux-arm64-testing-tests:
           filters:
             branches:
@@ -2403,7 +2416,7 @@ workflows:
             - linux-arm64-testing
       - linux-arm64-testing-gn-check:
           requires:
-            - linux-checkout-for-workspace
+            - linux-make-src-cache
 
   build-mac:
     when:
@@ -2412,14 +2425,13 @@ workflows:
         - equal: [false, << pipeline.parameters.run-linux-publish >>]
         - equal: [true, << pipeline.parameters.run-build-mac >>]
     jobs:
-      - mac-checkout-for-workspace
       - mac-make-src-cache
       - osx-testing-x64:
           requires:
             - mac-make-src-cache
       - osx-testing-x64-gn-check:
           requires:
-            - mac-checkout-for-workspace
+            - mac-make-src-cache
       - osx-testing-x64-tests:
           requires:
             - osx-testing-x64
@@ -2438,7 +2450,7 @@ workflows:
             - mac-make-src-cache
       - mas-testing-x64-gn-check:
           requires:
-            - mac-checkout-for-workspace
+            - mac-make-src-cache
       - mas-testing-x64-tests:
           requires:
             - mas-testing-x64

BUILD.gn

@@ -1231,6 +1231,10 @@ if (is_mac) {
       if (!is_component_build && is_component_ffmpeg) {
         configs += [ "//build/config/gcc:rpath_for_built_shared_libraries" ]
       }
+
+      if (is_linux) {
+        deps += [ "//sandbox/linux:chrome_sandbox" ]
+      }
     }
   }
 

script/node-disabled-tests.json

@@ -42,6 +42,7 @@
   "parallel/test-https-agent-session-reuse",
   "parallel/test-https-options-boolean-check",
   "parallel/test-inspector-inspect-brk-node",
+  "parallel/test-icu-minimum-version",
   "parallel/test-inspector-multisession-ws",
   "parallel/test-inspector-port-zero-cluster",
   "parallel/test-inspector-tracing-domain",