Split build and test into separate jobs

Also clean from review

.circleci/config.yml

@@ -419,7 +419,7 @@ steps-build-mac: &steps-build-mac
     # It would be better to verify ffmpeg as a part of a test job,
     # but it requires `gn` to run, and it's complicated
     # to store all gn's dependencies and configs.
-    - <<: *step-verify-ffmpeg
+    - *step-verify-ffmpeg
 
     # Node.js headers for tests
     - *step-nodejs-headers-build

BUILD.gn

@@ -906,7 +906,6 @@ dist_zip("electron_dist_zip") {
   ]
 }
 
-
 dist_zip("electron_ffmpeg_zip") {
   data_deps = [
     "//third_party/ffmpeg",

build/zip.py

@@ -21,12 +21,10 @@ PATHS_TO_SKIP = [
 ]
 
 def skip_path(dep):
-    should_skip = (
-        any(dep.startswith(path) for path in PATHS_TO_SKIP) or
-        any(dep.endswith(ext) for ext in EXTENSIONS_TO_SKIP))
-    if should_skip:
-      print("Skipping {}".format(dep))
-    return should_skip
+  should_skip = (
+    any(dep.startswith(path) for path in PATHS_TO_SKIP) or
+    any(dep.endswith(ext) for ext in EXTENSIONS_TO_SKIP))
+  return should_skip
 
 def strip_binaries(target_cpu, dep):
   for binary in LINUX_BINARIES_TO_STRIP:

vsts.yml

@@ -1,6 +1,7 @@
 jobs:
 - job: Build_Electron_via_GN
-  timeoutInMinutes: 180
+  displayName: Build Electron via GN
+  timeoutInMinutes: 120
   steps:
   - bash: |
       export PATH="$PATH:/Users/electron/depot_tools"
@@ -51,6 +52,12 @@ jobs:
       ninja -C out/Default electron:electron_app
     displayName: Ninja build app
 
+  - bash: |
+      cd src
+      ninja -C out/Default third_party/electron_node:headers
+    displayName: Build Node.js headers for testing
+    condition: and(succeeded(), eq(variables['RUN_TESTS'], '1'))
+
   - bash: |
       cd src
       gn gen out/ffmpeg --args='import("//electron/build/args/ffmpeg.gn") cc_wrapper="'"$SCCACHE_PATH"'"'" $GN_EXTRA_ARGS"
@@ -63,16 +70,6 @@ jobs:
     displayName: Check sccache stats after build
     condition: and(succeeded(), ne(variables['ELECTRON_RELEASE'], '1'))
 
-  - bash: |
-      cd src
-      # Make sure there aren't any Electron processes left running from previous tests
-      killall Electron
-      ninja -C out/Default third_party/electron_node:headers
-      export ELECTRON_OUT_DIR=Default
-      (cd electron && npm run test -- --ci --enable-logging)
-    displayName: Test
-    condition: and(succeeded(), eq(variables['RUN_TESTS'], '1'))
-
   - bash: |
       cd src
       ninja -C out/Default electron:electron_dist_zip
@@ -83,24 +80,109 @@ jobs:
       ninja -C out/Default electron:electron_chromedriver_zip
     displayName: Build chromedriver and zip
 
-  - task: PublishTestResults@2
-    displayName: Publish Test Results
-    inputs:
-      testResultsFiles: '*.xml'
-      searchFolder: '$(System.DefaultWorkingDirectory)/src/junit/'
-    condition: and(always(), eq(variables['MOCHA_FILE'], 'junit/test-results.xml'), eq(variables['RUN_TESTS'], '1'))
-
   - task: PublishBuildArtifacts@1
-    displayName: Publish Build Artifacts
+    displayName: Publish Build Artifacts (application zip)
     inputs:
       PathtoPublish: '$(System.DefaultWorkingDirectory)/src/out/Default/dist.zip'
-      ArtifactName: dist
+      ArtifactName: Default
 
   - task: PublishBuildArtifacts@1
-    displayName: Publish Build Artifacts
+    displayName: Publish Build Artifacts (chromedriver.zip)
     inputs:
       PathtoPublish: '$(System.DefaultWorkingDirectory)/src/out/Default/chromedriver.zip'
-      ArtifactName: dist
+      ArtifactName: Default
+
+  - task: PublishBuildArtifacts@1
+    displayName: Publish gn args for ffmpeg testing
+    inputs:
+      PathtoPublish: '$(System.DefaultWorkingDirectory)/src/out/Default/args.gn'
+      ArtifactName: Default
+
+  - task: PublishBuildArtifacts@1
+    displayName: Publish build.ninja for ffmpeg testing
+    inputs:
+      PathtoPublish: '$(System.DefaultWorkingDirectory)/src/out/Default/build.ninja'
+      ArtifactName: Default
+    condition: and(succeeded(), eq(variables['RUN_TESTS'], '1'))
+
+  - task: PublishBuildArtifacts@1
+    displayName: Publish libffmpeg.dylib for testing
+    inputs:
+      PathtoPublish: '$(System.DefaultWorkingDirectory)/src/out/ffmpeg/libffmpeg.dylib'
+      ArtifactName: ffmpeg
+    condition: and(succeeded(), eq(variables['RUN_TESTS'], '1'))
+
+  - task: PublishBuildArtifacts@1
+    displayName: Publish Node.js headers for testing
+    inputs:
+      PathtoPublish: '$(System.DefaultWorkingDirectory)/src/out/Default/gen/node_headers'
+      ArtifactName: node_headers
+    condition: and(succeeded(), eq(variables['RUN_TESTS'], '1'))
+
+- job: Test_Electron_Build
+  displayName: Test
+  dependsOn: Build_Electron_via_GN
+  condition: and(succeeded(), eq(variables['RUN_TESTS'], '1'))
+  timeoutInMinutes: 30
+  steps:
+  - bash: |
+      set -ex
+      mkdir -p src/out/Default/gen
+      mkdir -p src/out/ffmpeg
+      git clone https://github.com/electron/electron src/electron
+      (cd src/electron; git fetch origin +"${BUILD_SOURCEBRANCH}"; git checkout "${BUILD_SOURCEVERSION}")
+    displayName: Checkout Electron
+
+  - task: DownloadBuildArtifacts@0
+    displayName: 'Download files needed for testing'
+    inputs:
+      artifactName: Default
+      downloadPath: '$(System.DefaultWorkingDirectory)/src/out'
+
+  - task: DownloadBuildArtifacts@0
+    displayName: 'Download ffmpeg.dylib for testing'
+    inputs:
+      artifactName: ffmpeg
+      downloadPath: '$(System.DefaultWorkingDirectory)/src/out/ffmpeg'
+
+  - task: DownloadBuildArtifacts@0
+    displayName: 'Download Node.js headers'
+    inputs:
+      artifactName: node_headers
+      downloadPath: '$(System.DefaultWorkingDirectory)/src/out/Default/gen'
+
+  - bash: |
+      cd src/out/Default
+      unzip dist.zip
+    displayName: Unzip Electron app
+
+  - bash: |
+      set +e
+      killall Electron
+    displayName: Make sure Electron isn't running from previous tests
+
+  - bash: |
+      cd src
+      python electron/script/verify-ffmpeg.py --source-root "$PWD" --build-dir out/Default --ffmpeg-path out/ffmpeg
+    displayName: Verify non proprietary ffmpeg
+    condition: and(succeeded(), eq(variables['RUN_TESTS'], '1'))
+    timeoutInMinutes: 5
+
+  - bash: |
+      cd src
+      ninja -C out/Default third_party/electron_node:headers
+      export ELECTRON_OUT_DIR=Default
+      (cd electron && npm run test -- --ci --enable-logging)
+    displayName: Run Electron test suite
+    condition: and(succeeded(), eq(variables['RUN_TESTS'], '1'))
+    timeoutInMinutes: 10
+
+  - task: PublishTestResults@2
+    displayName: Publish Test Results
+    inputs:
+      testResultsFiles: '*.xml'
+      searchFolder: '$(System.DefaultWorkingDirectory)/src/junit/'
+    condition: and(always(), eq(variables['MOCHA_FILE'], 'junit/test-results.xml'))
 
   - bash: |
       export BUILD_URL="${SYSTEM_TEAMFOUNDATIONCOLLECTIONURI}${SYSTEM_TEAMPROJECT}/_build/results?buildId=${BUILD_BUILDID}"