ci: use goma for windows and linux builds (#21868)

* ci: use goma for windows and linux builds

.circleci/config.yml

@@ -53,7 +53,7 @@ parameters:
 # Build machines configs.
 docker-image: &docker-image
   docker:
-    - image: electronjs/build:697b894f36d127155e020f4e8ad4b2e5f6a09613
+    - image: electronjs/build:3bba0fdee0d5650e751c2561f1806f9138dfe56a
 
 machine-linux-medium: &machine-linux-medium
   <<: *docker-image
@@ -219,6 +219,15 @@ step-setup-env-for-build: &step-setup-env-for-build
     command: |
       # To find `gn` executable.
       echo 'export CHROMIUM_BUILDTOOLS_PATH="'"$PWD"'/src/buildtools"' >> $BASH_ENV
+      # Setup Goma if applicable
+      if [ ! -z "$RAW_GOMA_AUTH" ] && [ "`uname`" == "Linux" ]; then
+        export USE_SCCACHE="false"
+        echo 'export USE_SCCACHE=false' >> $BASH_ENV
+        echo 'export USE_GOMA=true' >> $BASH_ENV
+        echo 'export NUMBER_OF_NINJA_PROCESSES=300' >> $BASH_ENV
+        echo $RAW_GOMA_AUTH > ~/.goma_oauth2_config
+        src/electron/external_binaries/goma/goma_ctl.py ensure_start
+      fi
 
       if [ "$USE_SCCACHE" == "true" ]; then
         # https://github.com/mozilla/sccache
@@ -320,7 +329,11 @@ step-gn-gen-default: &step-gn-gen-default
     name: Default GN gen
     command: |
       cd src
-      gn gen out/Default --args='import("'$GN_CONFIG'") cc_wrapper="'"$SCCACHE_PATH"'"'" $GN_EXTRA_ARGS $GN_BUILDFLAG_ARGS"
+      if [ "$USE_GOMA" == "true" ]; then
+        gn gen out/Default --args="import(\"$GN_CONFIG\") import(\"//electron/build/args/goma.gn\") $GN_EXTRA_ARGS $GN_BUILDFLAG_ARGS"
+      else
+        gn gen out/Default --args="import(\"$GN_CONFIG\") cc_wrapper=\"$SCCACHE_PATH\" $GN_EXTRA_ARGS $GN_BUILDFLAG_ARGS"
+      fi
 
 step-gn-check: &step-gn-check
   run:
@@ -499,7 +512,11 @@ step-ffmpeg-gn-gen: &step-ffmpeg-gn-gen
     name: ffmpeg GN gen
     command: |
       cd src
-      gn gen out/ffmpeg --args='import("//electron/build/args/ffmpeg.gn") cc_wrapper="'"$SCCACHE_PATH"'"'" $GN_EXTRA_ARGS"
+      if [ "$USE_GOMA" == "true" ]; then
+        gn gen out/ffmpeg --args="import(\"//electron/build/args/ffmpeg.gn\") import(\"//electron/build/args/goma.gn\") $GN_EXTRA_ARGS"
+      else
+        gn gen out/ffmpeg --args="import(\"//electron/build/args/ffmpeg.gn\") cc_wrapper=\"$SCCACHE_PATH\" $GN_EXTRA_ARGS"
+      fi
 
 step-ffmpeg-build: &step-ffmpeg-build
   run:
@@ -537,11 +554,14 @@ step-setup-linux-for-headless-testing: &step-setup-linux-for-headless-testing
 
 step-show-sccache-stats: &step-show-sccache-stats
   run:
-    name: Check sccache stats after build
+    name: Check sccache/goma stats after build
     command: |
       if [ "$SCCACHE_PATH" != "" ]; then
         $SCCACHE_PATH -s
       fi
+      if [ "$USE_GOMA" == "true" ]; then
+        src/electron/external_binaries/goma/goma_ctl.py stat
+      fi
 
 step-mksnapshot-build: &step-mksnapshot-build
   run:
@@ -1264,6 +1284,9 @@ commands:
       build:
         type: boolean
         default: true
+      use-out-cache:
+        type: boolean
+        default: true
     steps:
       - when:
           condition: << parameters.attach >>
@@ -1323,7 +1346,10 @@ commands:
             - *step-gn-gen-default
 
             # Electron app
-            - *step-restore-out-cache
+            - when:
+                condition: << parameters.use-out-cache >>
+                steps:
+                  - *step-restore-out-cache
             - *step-gn-gen-default
             - *step-electron-build
             - *step-ninja-summary
@@ -1385,7 +1411,10 @@ commands:
                     # These files do not seem to like being in a cache, let us remove them
                     find . -type f -name '*_pkg_info' -delete
                   fi
-            - *step-save-out-cache
+            - when:
+                condition: << parameters.use-out-cache >>
+                steps:
+                  - *step-save-out-cache
 
       # Trigger tests on arm hardware if needed
       - *step-maybe-trigger-arm-test
@@ -1488,6 +1517,7 @@ jobs:
       - electron-build:
           persist: true
           checkout: true
+          use-out-cache: false
 
   linux-x64-testing-no-run-as-node:
     <<: *machine-linux-2xlarge
@@ -1502,6 +1532,7 @@ jobs:
       - electron-build:
           persist: false
           checkout: true
+          use-out-cache: false
 
   linux-x64-testing-gn-check:
     <<: *machine-linux-medium
@@ -1552,6 +1583,7 @@ jobs:
       - electron-build:
           persist: true
           checkout: true
+          use-out-cache: false
 
   linux-ia32-chromedriver:
     <<: *machine-linux-medium
@@ -1600,6 +1632,7 @@ jobs:
       - electron-build:
           persist: false
           checkout: true
+          use-out-cache: false
 
   linux-arm-chromedriver:
     <<: *machine-linux-medium
@@ -1648,6 +1681,7 @@ jobs:
       - electron-build:
           persist: false
           checkout: true
+          use-out-cache: false
 
   linux-arm64-testing-gn-check:
     <<: *machine-linux-medium

appveyor.yml

@@ -108,15 +108,21 @@ build_script:
           Write-warning "Could not save source to shared drive; continuing anyway"
         }
       }
+  - ps: >-
+      if (Test-Path 'env:RAW_GOMA_AUTH') {
+        $env:GOMA_OAUTH2_CONFIG_FILE = "$pwd\.goma_oauth2_config"
+        $env:RAW_GOMA_AUTH | Set-Content $env:GOMA_OAUTH2_CONFIG_FILE
+        .\src\electron\script\start-goma.ps1 -gomaDir "$pwd\src\electron\external_binaries\goma"
+      }
   - cd src
-  - ps: $env:BUILD_CONFIG_PATH="//electron/build/args/%GN_CONFIG%.gn"
-  - gn gen out/Default "--args=import(\"%BUILD_CONFIG_PATH%\") %GN_EXTRA_ARGS% cc_wrapper=\"%SCCACHE_PATH%\""
+  - set BUILD_CONFIG_PATH=//electron/build/args/%GN_CONFIG%.gn 
+  - if DEFINED RAW_GOMA_AUTH (gn gen out/Default "--args=import(\"%BUILD_CONFIG_PATH%\") import(\"//electron/build/args/goma.gn\") %GN_EXTRA_ARGS% ") else (gn gen out/Default "--args=import(\"%BUILD_CONFIG_PATH%\") %GN_EXTRA_ARGS% cc_wrapper=\"%SCCACHE_PATH%\"")
   - gn check out/Default //electron:electron_lib
   - gn check out/Default //electron:electron_app
   - gn check out/Default //electron:manifests
   - gn check out/Default //electron/shell/common/api:mojo
-  - ninja -C out/Default electron:electron_app
-  - if "%GN_CONFIG%"=="testing" ( python C:\Users\electron\depot_tools\post_build_ninja_summary.py -C out\Default )  
+  - if DEFINED RAW_GOMA_AUTH (ninja -j 300 -C out/Default electron:electron_app) else (ninja -C out/Default electron:electron_app)
+  - if "%GN_CONFIG%"=="testing" ( python C:\Users\electron\depot_tools\post_build_ninja_summary.py -C out\Default )
   - gn gen out/ffmpeg "--args=import(\"//electron/build/args/ffmpeg.gn\") %GN_EXTRA_ARGS%"
   - ninja -C out/ffmpeg electron:electron_ffmpeg_zip
   - ninja -C out/Default electron:electron_dist_zip
@@ -125,7 +131,7 @@ build_script:
   - ninja -C out/Default electron:hunspell_dictionaries_zip
   - ninja -C out/Default electron:electron_chromedriver_zip
   - ninja -C out/Default third_party/electron_node:headers
-  - cmd /C %SCCACHE_PATH% --show-stats
+  - if DEFINED RAW_GOMA_AUTH (python electron\external_binaries\goma\goma_ctl.py stat)
   - python electron/build/profile_toolchain.py --output-json=out/Default/windows_toolchain_profile.json
   - appveyor PushArtifact out/Default/windows_toolchain_profile.json
   - appveyor PushArtifact out/Default/dist.zip

script/start-goma.ps1

@@ -0,0 +1,11 @@
+param([string]$gomaDir=$PWD)
+$cmdPath = Join-Path -Path $gomaDir -ChildPath "goma_ctl.py" 
+Start-Process -FilePath cmd -ArgumentList "/C", "python", "$cmdPath", "ensure_start"
+$env:GLOG_logtostderr = "true"
+$timedOut = $false; $waitTime = 0; $waitIncrement = 5; $timeout=120;
+Do { sleep $waitIncrement; $timedOut = (($waitTime+=$waitIncrement) -gt $timeout); iex "$gomaDir\gomacc.exe port 2" > $null; } Until(($LASTEXITCODE -eq 0) -or $timedOut) 
+if ($timedOut) { 
+    write-error 'Timed out waiting for goma to start'; exit 1; 
+} else {
+    Write-Output "Successfully started goma!"
+}