fix: dump correct breakpad symbols on macOS (#19155)

.circleci/config.yml

@@ -427,25 +427,13 @@ step-mksnapshot-store: &step-mksnapshot-store
     path: src/out/Default/mksnapshot.zip
     destination: mksnapshot.zip
 
-step-maybe-build-dump-syms: &step-maybe-build-dump-syms
-  run:
-    name: Build dump_syms binary
-    command: |
-      if [ "$GENERATE_SYMBOLS" == "true" ]; then
-        cd src
-        # Build needed dump_syms executable
-        ninja -C out/Default third_party/breakpad:dump_syms
-      fi
-
 step-maybe-generate-breakpad-symbols: &step-maybe-generate-breakpad-symbols
   run:
     name: Generate breakpad symbols
     command: |
       if [ "$GENERATE_SYMBOLS" == "true" ]; then
         cd src
-        export BUILD_PATH="$PWD/out/Default"
-        export DEST_PATH="$BUILD_PATH/breakpad_symbols"
-        electron/script/dump-symbols.py -b $BUILD_PATH -d $DEST_PATH -v
+        ninja -C out/Default electron:electron_symbols
       fi
 
 step-maybe-zip-symbols: &step-maybe-zip-symbols
@@ -653,7 +641,6 @@ steps-electron-build-for-tests: &steps-electron-build-for-tests
     # Save all data needed for a further tests run.
     - *step-persist-data-for-tests
 
-    - *step-maybe-build-dump-syms
     - *step-maybe-generate-breakpad-symbols
     - *step-maybe-zip-symbols
 
@@ -679,7 +666,6 @@ steps-electron-build-for-publish: &steps-electron-build-for-publish
     - *step-maybe-electron-dist-strip
     - *step-electron-dist-build
     - *step-electron-dist-store
-    - *step-maybe-build-dump-syms
     - *step-maybe-generate-breakpad-symbols
     - *step-maybe-zip-symbols
 

BUILD.gn

@@ -10,6 +10,7 @@ import("//tools/grit/repack.gni")
 import("//tools/v8_context_snapshot/v8_context_snapshot.gni")
 import("//v8/gni/snapshot_toolchain.gni")
 import("build/asar.gni")
+import("build/extract_symbols.gni")
 import("build/js_wrap.gni")
 import("build/npm.gni")
 import("build/tsc.gni")
@@ -712,6 +713,7 @@ if (is_mac) {
   electron_helper_name = "$electron_product_name Helper"
   electron_login_helper_name = "$electron_product_name Login Helper"
   electron_framework_version = "A"
+  electron_version = read_file("ELECTRON_VERSION", "trim string")
 
   mac_xib_bundle_data("electron_xibs") {
     sources = [
@@ -840,7 +842,6 @@ if (is_mac) {
     }
     info_plist = "atom/common/resources/mac/Info.plist"
 
-    electron_version = read_file("ELECTRON_VERSION", "trim string")
     extra_substitutions = [
       "ATOM_BUNDLE_ID=$electron_mac_bundle_id.framework",
       "ELECTRON_VERSION=$electron_version",
@@ -1014,6 +1015,76 @@ if (is_mac) {
       "@executable_path/../Frameworks",
     ]
   }
+
+  if (enable_dsyms) {
+    extract_symbols("electron_framework_syms") {
+      binary = "$root_out_dir/$electron_framework_name.framework/Versions/$electron_framework_version/$electron_framework_name"
+      symbol_dir = "$root_out_dir/breakpad_symbols"
+      dsym_file = "$root_out_dir/$electron_framework_name.dSYM/Contents/Resources/DWARF/$electron_framework_name"
+      deps = [
+        ":electron_framework",
+      ]
+    }
+
+    extract_symbols("electron_helper_syms") {
+      binary = "$root_out_dir/$electron_helper_name.app/Contents/MacOS/$electron_helper_name"
+      symbol_dir = "$root_out_dir/breakpad_symbols"
+      dsym_file = "$root_out_dir/$electron_helper_name.dSYM/Contents/Resources/DWARF/$electron_helper_name"
+      deps = [
+        ":electron_helper_app",
+      ]
+    }
+
+    extract_symbols("electron_app_syms") {
+      binary = "$root_out_dir/$electron_product_name.app/Contents/MacOS/$electron_product_name"
+      symbol_dir = "$root_out_dir/breakpad_symbols"
+      dsym_file = "$root_out_dir/$electron_product_name.dSYM/Contents/Resources/DWARF/$electron_product_name"
+      deps = [
+        ":electron_app",
+      ]
+    }
+
+    extract_symbols("swiftshader_egl_syms") {
+      binary = "$root_out_dir/libswiftshader_libEGL.dylib"
+      symbol_dir = "$root_out_dir/breakpad_symbols"
+      dsym_file = "$root_out_dir/libswiftshader_libEGL.dylib.dSYM/Contents/Resources/DWARF/libswiftshader_libEGL.dylib"
+      deps = [
+        "//third_party/swiftshader/src/OpenGL/libEGL:swiftshader_libEGL",
+      ]
+    }
+
+    extract_symbols("swiftshader_gles_syms") {
+      binary = "$root_out_dir/libswiftshader_libGLESv2.dylib"
+      symbol_dir = "$root_out_dir/breakpad_symbols"
+      dsym_file = "$root_out_dir/libswiftshader_libGLESv2.dylib.dSYM/Contents/Resources/DWARF/libswiftshader_libGLESv2.dylib"
+      deps = [
+        "//third_party/swiftshader/src/OpenGL/libGLESv2:swiftshader_libGLESv2",
+      ]
+    }
+
+    extract_symbols("crashpad_handler_syms") {
+      binary = "$root_out_dir/crashpad_handler"
+      symbol_dir = "$root_out_dir/breakpad_symbols"
+      dsym_file = "$root_out_dir/crashpad_handler.dSYM/Contents/Resources/DWARF/crashpad_handler"
+      deps = [
+        "//third_party/crashpad/crashpad/handler:crashpad_handler",
+      ]
+    }
+
+    group("electron_symbols") {
+      deps = [
+        ":crashpad_handler_syms",
+        ":electron_app_syms",
+        ":electron_framework_syms",
+        ":electron_helper_syms",
+        ":swiftshader_egl_syms",
+        ":swiftshader_gles_syms",
+      ]
+    }
+  } else {
+    group("electron_symbols") {
+    }
+  }
 } else {
   windows_manifest("electron_app_manifest") {
     sources = [
@@ -1102,6 +1173,49 @@ if (is_mac) {
       }
     }
   }
+
+  if (is_official_build) {
+    if (is_linux) {
+      _target_executable_suffix = ""
+      _target_shared_library_suffix = ".so"
+    } else if (is_win) {
+      _target_executable_suffix = ".exe"
+      _target_shared_library_suffix = ".dll"
+    }
+
+    extract_symbols("electron_app_symbols") {
+      binary = "$root_out_dir/$electron_project_name$_target_executable_suffix"
+      symbol_dir = "$root_out_dir/breakpad_symbols"
+      deps = [
+        ":electron_app",
+      ]
+    }
+
+    extract_symbols("swiftshader_egl_symbols") {
+      binary = "$root_out_dir/swiftshader/libEGL$_target_shared_library_suffix"
+      symbol_dir = "$root_out_dir/breakpad_symbols"
+      deps = [
+        "//third_party/swiftshader/src/OpenGL/libEGL:swiftshader_libEGL",
+      ]
+    }
+
+    extract_symbols("swiftshader_gles_symbols") {
+      binary =
+          "$root_out_dir/swiftshader/libGLESv2$_target_shared_library_suffix"
+      symbol_dir = "$root_out_dir/breakpad_symbols"
+      deps = [
+        "//third_party/swiftshader/src/OpenGL/libGLESv2:swiftshader_libGLESv2",
+      ]
+    }
+
+    group("electron_symbols") {
+      deps = [
+        ":electron_app_symbols",
+        ":swiftshader_egl_symbols",
+        ":swiftshader_gles_symbols",
+      ]
+    }
+  }
 }
 
 template("dist_zip") {

appveyor.yml

@@ -72,9 +72,10 @@ build_script:
   - appveyor PushArtifact out/ffmpeg/ffmpeg.zip
   - ps: >-
       if ($env:GN_CONFIG -eq 'release') {
-        ninja -C out/Default third_party/breakpad:dump_syms
+        # Needed for msdia140.dll on 64-bit windows
+        $env:Path += ";$pwd\third_party\llvm-build\Release+Asserts\bin"
+        ninja -C out/Default electron:electron_symbols
       }
-  - if "%GN_CONFIG%"=="release" ( python electron\script\dump-symbols.py -d %cd%\out\Default\breakpad_symbols -v)
   - ps: >-
       if ($env:GN_CONFIG -eq 'release') {
         python electron\script\zip-symbols.py

build/dump_syms.py

@@ -0,0 +1,52 @@
+from __future__ import print_function
+
+import collections
+import os
+import subprocess
+import sys
+import errno
+
+# The BINARY_INFO tuple describes a binary as dump_syms identifies it.
+BINARY_INFO = collections.namedtuple('BINARY_INFO',
+                                     ['platform', 'arch', 'hash', 'name'])
+
+def get_module_info(header_info):
+  # header info is of the form "MODULE $PLATFORM $ARCH $HASH $BINARY"
+  info_split = header_info.strip().split(' ', 4)
+  if len(info_split) != 5 or info_split[0] != 'MODULE':
+    return None
+  return BINARY_INFO(*info_split[1:])
+
+def get_symbol_path(symbol_data):
+  module_info = get_module_info(symbol_data[:symbol_data.index('\n')])
+  if not module_info:
+    raise Exception("Couldn't get module info for binary '{}'".format(binary))
+  return os.path.join(module_info.name, module_info.hash, module_info.name + ".sym")
+
+def mkdir_p(path):
+  """Simulates mkdir -p."""
+  try:
+    os.makedirs(path)
+  except OSError as e:
+    if e.errno == errno.EEXIST and os.path.isdir(path):
+      pass
+    else: raise
+
+def main(dump_syms, binary, out_dir, stamp_file, dsym_file=None):
+  args = [dump_syms]
+  if dsym_file:
+    args += ["-g", dsym_file]
+  args += [binary]
+
+  symbol_data = subprocess.check_output(args)
+  symbol_path = os.path.join(out_dir, get_symbol_path(symbol_data))
+  mkdir_p(os.path.dirname(symbol_path))
+
+  with open(symbol_path, 'w') as out:
+    out.write(symbol_data)
+
+  with open(stamp_file, 'w'):
+    pass
+
+if __name__ == '__main__':
+  main(*sys.argv[1:])

build/extract_symbols.gni

@@ -0,0 +1,55 @@
+import("//build/toolchain/toolchain.gni")
+
+# Extracts symbols from a binary into a symbol file using dump_syms.
+#
+# Args:
+#   binary: Path to the binary containing symbols to extract, e.g.:
+#       "$root_out_dir/electron"
+#   symbol_dir: Desired output directory for symbols, e.g.:
+#       "$root_out_dir/breakpad_symbols"
+
+if (host_os == "win") {
+  _host_executable_suffix = ".exe"
+} else {
+  _host_executable_suffix = ""
+}
+
+template("extract_symbols") {
+  action(target_name) {
+    forward_variables_from(invoker,
+                           [
+                             "deps",
+                             "testonly",
+                           ])
+    assert(defined(invoker.binary), "Need binary to dump")
+    assert(defined(invoker.symbol_dir), "Need directory for symbol output")
+
+    dump_syms_label = "//third_party/breakpad:dump_syms($host_toolchain)"
+    dump_syms_binary = get_label_info(dump_syms_label, "root_out_dir") +
+                       "/dump_syms$_host_executable_suffix"
+
+    script = "//electron/build/dump_syms.py"
+    inputs = [
+      invoker.binary,
+      dump_syms_binary,
+    ]
+    stamp_file = "${target_gen_dir}/${target_name}.stamp"
+    outputs = [
+      stamp_file,
+    ]
+    args = [
+      "./" + rebase_path(dump_syms_binary, root_build_dir),
+      rebase_path(invoker.binary, root_build_dir),
+      rebase_path(invoker.symbol_dir, root_build_dir),
+      rebase_path(stamp_file, root_build_dir),
+    ]
+    if (defined(invoker.dsym_file)) {
+      args += [ rebase_path(invoker.dsym_file, root_build_dir) ]
+    }
+
+    if (!defined(deps)) {
+      deps = []
+    }
+    deps += [ dump_syms_label ]
+  }
+}

script/dump-symbols.py

@@ -1,98 +0,0 @@
-#!/usr/bin/env python
-
-import argparse
-import os
-import sys
-
-from lib.config import PLATFORM, enable_verbose_mode, is_verbose_mode
-from lib.util import get_electron_branding, execute, rm_rf, get_out_dir, \
-                     SRC_DIR
-
-ELECTRON_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
-SOURCE_ROOT = os.path.abspath(os.path.dirname(ELECTRON_ROOT))
-RELEASE_PATH = get_out_dir()
-
-def main():
-  args = parse_args()
-  if args.verbose:
-    enable_verbose_mode()
-  rm_rf(args.destination)
-  source_root = os.path.abspath(args.source_root)
-  build_path = os.path.join(source_root, args.build_dir)
-  (project_name, product_name) = get_names_from_branding()
-
-  if PLATFORM in ['darwin', 'linux']:
-
-    if PLATFORM == 'darwin':
-      #macOS has an additional helper app; provide the path to that binary also
-      main_app = os.path.join(build_path, '{0}.app'.format(product_name),
-                            'Contents', 'MacOS', product_name)
-      helper_name = product_name + " Helper"
-      helper_app = os.path.join(build_path, '{0}.app'.format(helper_name),
-                            'Contents', 'MacOS', product_name + " Helper")
-      binaries = [main_app, helper_app]
-      for binary in binaries:
-        generate_posix_symbols(binary, source_root, build_path,
-                                        args.destination)
-    else:
-      binary = os.path.join(build_path, project_name)
-      generate_posix_symbols(binary, source_root, build_path,
-                                      args.destination)
-
-  else:
-    generate_breakpad_symbols = os.path.join(ELECTRON_ROOT, 'tools', 'win',
-                                             'generate_breakpad_symbols.py')
-    args = [
-      '--symbols-dir={0}'.format(args.destination),
-      '--jobs=16',
-      os.path.relpath(build_path),
-    ]
-    if is_verbose_mode():
-      args += ['-v']
-    #Make sure msdia140.dll is in the path (needed for dump_syms.exe)
-    env = os.environ.copy()
-    msdia140_dll_path =  os.path.join(SRC_DIR, 'third_party', 'llvm-build',
-                                      'Release+Asserts', 'bin')
-    env['PATH'] = os.path.pathsep.join(
-        [env.get('PATH', '')] + [msdia140_dll_path])
-    execute([sys.executable, generate_breakpad_symbols] + args, env)
-
-def get_names_from_branding():
-  variables = get_electron_branding()
-  return (variables['project_name'], variables['product_name'])
-
-def generate_posix_symbols(binary, source_root, build_dir, destination):
-  generate_breakpad_symbols = os.path.join(source_root, 'components', 'crash',
-                                          'content', 'tools',
-                                          'generate_breakpad_symbols.py')
-  args = [
-    '--build-dir={0}'.format(build_dir),
-    '--symbols-dir={0}'.format(destination),
-    '--jobs=16',
-    '--binary={0}'.format(binary),
-  ]
-  if is_verbose_mode():
-    args += ['--verbose']
-  execute([sys.executable, generate_breakpad_symbols] + args)
-
-def parse_args():
-  parser = argparse.ArgumentParser(description='Create breakpad symbols')
-  parser.add_argument('-b', '--build-dir',
-                      help='Path to an Electron build folder.',
-                      default=RELEASE_PATH,
-                      required=False)
-  parser.add_argument('-d', '--destination',
-                      help='Path to save symbols to.',
-                      default=None,
-                      required=True)
-  parser.add_argument('-s', '--source-root',
-                      help='Path to the src folder.',
-                      default=SOURCE_ROOT,
-                      required=False)
-  parser.add_argument('-v', '--verbose',
-                      action='store_true',
-                      help='Prints the output of the subprocesses')
-  return parser.parse_args()
-
-if __name__ == '__main__':
-  sys.exit(main())

tools/win/generate_breakpad_symbols.py

@@ -1,138 +0,0 @@
-#!/usr/bin/env python
-# Copyright (c) 2013 GitHub, Inc.
-# Copyright (c) 2013 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""Convert pdb to sym for given directories"""
-
-import errno
-import glob
-import optparse
-import os
-import Queue
-import re
-import subprocess
-import sys
-import threading
-
-SRC_DIR = os.path.abspath(os.path.join(__file__, '..', '..', '..', '..'))
-
-# Duplicated as this script lives in tools not script
-def get_out_dir():
-  out_dir = 'Debug'
-  override = os.environ.get('ELECTRON_OUT_DIR')
-  if override is not None:
-    out_dir = override
-  return os.path.join(SRC_DIR, 'out', out_dir)
-
-
-CONCURRENT_TASKS=4
-OUT_DIR=get_out_dir()
-DUMP_SYMS=os.path.join(OUT_DIR, 'dump_syms.exe')
-
-
-def GetCommandOutput(command):
-  """Runs the command list, returning its output.
-
-  Prints the given command (which should be a list of one or more strings),
-  then runs it and returns its output (stdout) as a string.
-
-  From chromium_utils.
-  """
-  devnull = open(os.devnull, 'w')
-  proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=devnull,
-                          bufsize=1)
-  output = proc.communicate()[0]
-  return output
-
-
-def mkdir_p(path):
-  """Simulates mkdir -p."""
-  try:
-    os.makedirs(path)
-  except OSError as e:
-    if e.errno == errno.EEXIST and os.path.isdir(path):
-      pass
-    else: raise
-
-
-def GenerateSymbols(options, binaries):
-  """Dumps the symbols of binary and places them in the given directory."""
-
-  queue = Queue.Queue()
-  print_lock = threading.Lock()
-
-  def _Worker():
-    while True:
-      binary = queue.get()
-
-      if options.verbose:
-        with print_lock:
-          print "Generating symbols for %s" % binary
-
-      syms = GetCommandOutput([DUMP_SYMS, binary])
-      module_line = re.match("MODULE [^ ]+ [^ ]+ ([0-9A-Fa-f]+) (.*)\r\n", syms)
-      if module_line == None:
-        with print_lock:
-          print "Failed to get symbols for %s" % binary
-        queue.task_done()
-        continue
-
-      output_path = os.path.join(options.symbols_dir, module_line.group(2),
-                                 module_line.group(1))
-      mkdir_p(output_path)
-      symbol_file = "%s.sym" % module_line.group(2)[:-4]  # strip .pdb
-      f = open(os.path.join(output_path, symbol_file), 'w')
-      f.write(syms)
-      f.close()
-
-      queue.task_done()
-
-  for binary in binaries:
-    queue.put(binary)
-
-  for _ in range(options.jobs):
-    t = threading.Thread(target=_Worker)
-    t.daemon = True
-    t.start()
-
-  queue.join()
-
-
-def main():
-  parser = optparse.OptionParser()
-  parser.add_option('', '--symbols-dir', default='',
-                    help='The directory where to write the symbols file.')
-  parser.add_option('', '--clear', default=False, action='store_true',
-                    help='Clear the symbols directory before writing new '
-                         'symbols.')
-  parser.add_option('-j', '--jobs', default=CONCURRENT_TASKS, action='store',
-                    type='int', help='Number of parallel tasks to run.')
-  parser.add_option('-v', '--verbose', action='store_true',
-                    help='Print verbose status output.')
-
-  (options, directories) = parser.parse_args()
-
-  if not options.symbols_dir:
-    print "Required option --symbols-dir missing."
-    return 1
-
-  if options.clear:
-    try:
-      shutil.rmtree(options.symbols_dir)
-    except:
-      pass
-
-  pdbs = []
-  for directory in directories:
-    pdbs += glob.glob(os.path.join(directory, '*.exe.pdb'))
-    pdbs += glob.glob(os.path.join(directory, '*.dll.pdb'))
-
-  GenerateSymbols(options, pdbs)
-
-  return 0
-
-
-if '__main__' == __name__:
-  sys.exit(main())