feat: serialize NativeImage over ipc (#30729)

filenames.auto.gni

@@ -138,7 +138,6 @@ auto_filenames = {
     "lib/common/api/native-image.ts",
     "lib/common/define-properties.ts",
     "lib/common/ipc-messages.ts",
-    "lib/common/type-utils.ts",
     "lib/common/web-view-methods.ts",
     "lib/renderer/api/context-bridge.ts",
     "lib/renderer/api/crash-reporter.ts",
@@ -168,7 +167,6 @@ auto_filenames = {
   ]
 
   isolated_bundle_deps = [
-    "lib/common/type-utils.ts",
     "lib/common/web-view-methods.ts",
     "lib/isolated_renderer/init.ts",
     "lib/renderer/web-view/web-view-attributes.ts",
@@ -246,7 +244,6 @@ auto_filenames = {
     "lib/common/ipc-messages.ts",
     "lib/common/parse-features-string.ts",
     "lib/common/reset-search-paths.ts",
-    "lib/common/type-utils.ts",
     "lib/common/web-view-events.ts",
     "lib/common/web-view-methods.ts",
     "lib/common/webpack-globals-provider.ts",
@@ -269,7 +266,6 @@ auto_filenames = {
     "lib/common/init.ts",
     "lib/common/ipc-messages.ts",
     "lib/common/reset-search-paths.ts",
-    "lib/common/type-utils.ts",
     "lib/common/web-view-methods.ts",
     "lib/common/webpack-provider.ts",
     "lib/renderer/api/context-bridge.ts",
@@ -309,7 +305,6 @@ auto_filenames = {
     "lib/common/init.ts",
     "lib/common/ipc-messages.ts",
     "lib/common/reset-search-paths.ts",
-    "lib/common/type-utils.ts",
     "lib/common/webpack-provider.ts",
     "lib/renderer/api/context-bridge.ts",
     "lib/renderer/api/crash-reporter.ts",

lib/browser/guest-view-manager.ts

@@ -4,7 +4,6 @@ import * as ipcMainUtils from '@electron/internal/browser/ipc-main-internal-util
 import { parseWebViewWebPreferences } from '@electron/internal/common/parse-features-string';
 import { syncMethods, asyncMethods, properties } from '@electron/internal/common/web-view-methods';
 import { webViewEvents } from '@electron/internal/common/web-view-events';
-import { serialize } from '@electron/internal/common/type-utils';
 import { IPC_MESSAGES } from '@electron/internal/common/ipc-messages';
 
 interface GuestInstance {
@@ -330,12 +329,6 @@ handleMessageSync(IPC_MESSAGES.GUEST_VIEW_MANAGER_PROPERTY_SET, function (event,
   (guest as any)[property] = val;
 });
 
-handleMessage(IPC_MESSAGES.GUEST_VIEW_MANAGER_CAPTURE_PAGE, async function (event, guestInstanceId: number, args: any[]) {
-  const guest = getGuestForWebContents(guestInstanceId, event.sender);
-
-  return serialize(await guest.capturePage(...args));
-});
-
 // Returns WebContents from its guest id hosted in given webContents.
 const getGuestForWebContents = function (guestInstanceId: number, contents: Electron.WebContents) {
   const guestInstance = guestInstances.get(guestInstanceId);

lib/browser/rpc-server.ts

@@ -4,7 +4,6 @@ import { clipboard } from 'electron/common';
 import * as fs from 'fs';
 import { ipcMainInternal } from '@electron/internal/browser/ipc-main-internal';
 import * as ipcMainUtils from '@electron/internal/browser/ipc-main-internal-utils';
-import * as typeUtils from '@electron/internal/common/type-utils';
 import { IPC_MESSAGES } from '@electron/internal/common/ipc-messages';
 
 import type * as desktopCapturerModule from '@electron/internal/browser/desktop-capturer';
@@ -56,7 +55,7 @@ ipcMainUtils.handleSync(IPC_MESSAGES.BROWSER_CLIPBOARD_SYNC, function (event, me
     throw new Error(`Invalid method: ${method}`);
   }
 
-  return typeUtils.serialize((clipboard as any)[method](...typeUtils.deserialize(args)));
+  return (clipboard as any)[method](...args);
 });
 
 if (BUILDFLAG(ENABLE_DESKTOP_CAPTURER)) {
@@ -71,7 +70,7 @@ if (BUILDFLAG(ENABLE_DESKTOP_CAPTURER)) {
       return [];
     }
 
-    return typeUtils.serialize(await desktopCapturer.getSourcesImpl(event.sender, options));
+    return await desktopCapturer.getSourcesImpl(event.sender, options);
   });
 }
 

lib/common/api/clipboard.ts

@@ -1,20 +1,14 @@
 import { IPC_MESSAGES } from '@electron/internal/common/ipc-messages';
 
 import type * as ipcRendererUtilsModule from '@electron/internal/renderer/ipc-renderer-internal-utils';
-import type * as typeUtilsModule from '@electron/internal/common/type-utils';
 
 const clipboard = process._linkedBinding('electron_common_clipboard');
 
 if (process.type === 'renderer') {
   const ipcRendererUtils = require('@electron/internal/renderer/ipc-renderer-internal-utils') as typeof ipcRendererUtilsModule;
-  const typeUtils = require('@electron/internal/common/type-utils') as typeof typeUtilsModule;
 
-  const makeRemoteMethod = function (method: keyof Electron.Clipboard) {
-    return (...args: any[]) => {
-      args = typeUtils.serialize(args);
-      const result = ipcRendererUtils.invokeSync(IPC_MESSAGES.BROWSER_CLIPBOARD_SYNC, method, ...args);
-      return typeUtils.deserialize(result);
-    };
+  const makeRemoteMethod = function (method: keyof Electron.Clipboard): any {
+    return (...args: any[]) => ipcRendererUtils.invokeSync(IPC_MESSAGES.BROWSER_CLIPBOARD_SYNC, method, ...args);
   };
 
   if (process.platform === 'linux') {

lib/common/ipc-messages.ts

@@ -13,7 +13,6 @@ export const enum IPC_MESSAGES {
   GUEST_VIEW_MANAGER_DETACH_GUEST = 'GUEST_VIEW_MANAGER_DETACH_GUEST',
   GUEST_VIEW_MANAGER_FOCUS_CHANGE = 'GUEST_VIEW_MANAGER_FOCUS_CHANGE',
   GUEST_VIEW_MANAGER_CALL = 'GUEST_VIEW_MANAGER_CALL',
-  GUEST_VIEW_MANAGER_CAPTURE_PAGE = 'GUEST_VIEW_MANAGER_CAPTURE_PAGE',
   GUEST_VIEW_MANAGER_PROPERTY_GET = 'GUEST_VIEW_MANAGER_PROPERTY_GET',
   GUEST_VIEW_MANAGER_PROPERTY_SET = 'GUEST_VIEW_MANAGER_PROPERTY_SET',
 

lib/common/type-utils.ts

@@ -1,110 +0,0 @@
-function getCreateNativeImage () {
-  return process._linkedBinding('electron_common_native_image').nativeImage.createEmpty;
-}
-
-export function isPromise (val: any) {
-  return (
-    val &&
-    val.then &&
-    val.then instanceof Function &&
-    val.constructor &&
-    val.constructor.reject &&
-    val.constructor.reject instanceof Function &&
-    val.constructor.resolve &&
-    val.constructor.resolve instanceof Function
-  );
-}
-
-const serializableTypes = [
-  Boolean,
-  Number,
-  String,
-  Date,
-  Error,
-  RegExp,
-  ArrayBuffer
-];
-
-// https://developer.mozilla.org/en-US/docs/Web/API/Web_Workers_API/Structured_clone_algorithm#Supported_types
-export function isSerializableObject (value: any) {
-  return value === null || ArrayBuffer.isView(value) || serializableTypes.some(type => value instanceof type);
-}
-
-const objectMap = function (source: Object, mapper: (value: any) => any) {
-  const sourceEntries = Object.entries(source);
-  const targetEntries = sourceEntries.map(([key, val]) => [key, mapper(val)]);
-  return Object.fromEntries(targetEntries);
-};
-
-function serializeNativeImage (image: Electron.NativeImage) {
-  const representations = [];
-  const scaleFactors = image.getScaleFactors();
-
-  // Use Buffer when there's only one representation for better perf.
-  // This avoids compressing to/from PNG where it's not necessary to
-  // ensure uniqueness of dataURLs (since there's only one).
-  if (scaleFactors.length === 1) {
-    const scaleFactor = scaleFactors[0];
-    const size = image.getSize(scaleFactor);
-    const buffer = image.toBitmap({ scaleFactor });
-    representations.push({ scaleFactor, size, buffer });
-  } else {
-    // Construct from dataURLs to ensure that they are not lost in creation.
-    for (const scaleFactor of scaleFactors) {
-      const size = image.getSize(scaleFactor);
-      const dataURL = image.toDataURL({ scaleFactor });
-      representations.push({ scaleFactor, size, dataURL });
-    }
-  }
-  return { __ELECTRON_SERIALIZED_NativeImage__: true, representations };
-}
-
-function deserializeNativeImage (value: any, createNativeImage: typeof Electron.nativeImage['createEmpty']) {
-  const image = createNativeImage();
-
-  // Use Buffer when there's only one representation for better perf.
-  // This avoids compressing to/from PNG where it's not necessary to
-  // ensure uniqueness of dataURLs (since there's only one).
-  if (value.representations.length === 1) {
-    const { buffer, size, scaleFactor } = value.representations[0];
-    const { width, height } = size;
-    image.addRepresentation({ buffer, scaleFactor, width, height });
-  } else {
-    // Construct from dataURLs to ensure that they are not lost in creation.
-    for (const rep of value.representations) {
-      const { dataURL, size, scaleFactor } = rep;
-      const { width, height } = size;
-      image.addRepresentation({ dataURL, scaleFactor, width, height });
-    }
-  }
-
-  return image;
-}
-
-export function serialize (value: any): any {
-  if (value && value.constructor && value.constructor.name === 'NativeImage') {
-    return serializeNativeImage(value);
-  } if (Array.isArray(value)) {
-    return value.map(serialize);
-  } else if (isSerializableObject(value)) {
-    return value;
-  } else if (value instanceof Object) {
-    return objectMap(value, serialize);
-  } else {
-    return value;
-  }
-}
-
-export function deserialize (value: any, createNativeImage: typeof Electron.nativeImage['createEmpty'] = getCreateNativeImage()): any {
-  if (value && value.__ELECTRON_SERIALIZED_NativeImage__) {
-    return deserializeNativeImage(value, createNativeImage);
-  } else if (Array.isArray(value)) {
-    return value.map(value => deserialize(value, createNativeImage));
-  } else if (isSerializableObject(value)) {
-    return value;
-  } else if (value instanceof Object) {
-    return objectMap(value, value => deserialize(value, createNativeImage));
-  } else {
-    return value;
-  }
-}

lib/common/web-view-methods.ts

@@ -59,6 +59,7 @@ export const properties = new Set([
 ]);
 
 export const asyncMethods = new Set([
+  'capturePage',
   'loadURL',
   'executeJavaScript',
   'insertCSS',

lib/renderer/api/desktop-capturer.ts

@@ -1,5 +1,4 @@
 import { ipcRendererInternal } from '@electron/internal/renderer/ipc-renderer-internal';
-import { deserialize } from '@electron/internal/common/type-utils';
 import deprecate from '@electron/internal/common/api/deprecate';
 import { IPC_MESSAGES } from '@electron/internal/common/ipc-messages';
 
@@ -21,5 +20,5 @@ export async function getSources (options: Electron.SourcesOptions) {
     deprecate.log('The use of \'desktopCapturer.getSources\' in the renderer process is deprecated and will be removed. See https://www.electronjs.org/docs/breaking-changes#removed-desktopcapturergetsources-in-the-renderer for more details.');
     warned = true;
   }
-  return deserialize(await ipcRendererInternal.invoke(IPC_MESSAGES.DESKTOP_CAPTURER_GET_SOURCES, options, getCurrentStack()));
+  return await ipcRendererInternal.invoke(IPC_MESSAGES.DESKTOP_CAPTURER_GET_SOURCES, options, getCurrentStack());
 }

lib/renderer/web-view/guest-view-internal.ts

@@ -43,10 +43,6 @@ export function detachGuest (guestInstanceId: number) {
   return ipcRendererUtils.invokeSync(IPC_MESSAGES.GUEST_VIEW_MANAGER_DETACH_GUEST, guestInstanceId);
 }
 
-export function capturePage (guestInstanceId: number, args: any[]) {
-  return ipcRendererInternal.invoke(IPC_MESSAGES.GUEST_VIEW_MANAGER_CAPTURE_PAGE, guestInstanceId, args);
-}
-
 export function invoke (guestInstanceId: number, method: string, args: any[]) {
   return ipcRendererInternal.invoke(IPC_MESSAGES.GUEST_VIEW_MANAGER_CALL, guestInstanceId, method, args);
 }

lib/renderer/web-view/web-view-impl.ts

@@ -3,7 +3,6 @@ import { WEB_VIEW_CONSTANTS } from '@electron/internal/renderer/web-view/web-vie
 import { syncMethods, asyncMethods, properties } from '@electron/internal/common/web-view-methods';
 import type { WebViewAttribute, PartitionAttribute } from '@electron/internal/renderer/web-view/web-view-attributes';
 import { setupWebViewAttributes } from '@electron/internal/renderer/web-view/web-view-attributes';
-import { deserialize } from '@electron/internal/common/type-utils';
 
 // ID generator.
 let nextId = 0;
@@ -16,7 +15,6 @@ export interface WebViewImplHooks {
   readonly guestViewInternal: typeof guestViewInternalModule;
   readonly allowGuestViewElementDefinition: NodeJS.InternalWebFrame['allowGuestViewElementDefinition'];
   readonly setIsWebView: (iframe: HTMLIFrameElement) => void;
-  readonly createNativeImage?: typeof Electron.nativeImage['createEmpty'];
 }
 
 // Represents the internal state of the WebView node.
@@ -235,10 +233,6 @@ export const setupMethods = (WebViewElement: typeof ElectronInternal.WebViewElem
     };
   }
 
-  WebViewElement.prototype.capturePage = async function (...args) {
-    return deserialize(await hooks.guestViewInternal.capturePage(this.getWebContentsId(), args), hooks.createNativeImage);
-  };
-
   const createPropertyGetter = function (property: string) {
     return function (this: ElectronInternal.WebViewElement) {
       return hooks.guestViewInternal.propertyGet(this.getWebContentsId(), property);

shell/common/api/electron_api_native_image.h

@@ -45,6 +45,12 @@ namespace api {
 
 class NativeImage : public gin::Wrappable<NativeImage> {
  public:
+  NativeImage(v8::Isolate* isolate, const gfx::Image& image);
+#if defined(OS_WIN)
+  NativeImage(v8::Isolate* isolate, const base::FilePath& hicon_path);
+#endif
+  ~NativeImage() override;
+
   static gin::Handle<NativeImage> CreateEmpty(v8::Isolate* isolate);
   static gin::Handle<NativeImage> Create(v8::Isolate* isolate,
                                          const gfx::Image& image);
@@ -95,13 +101,6 @@ class NativeImage : public gin::Wrappable<NativeImage> {
 
   const gfx::Image& image() const { return image_; }
 
- protected:
-  NativeImage(v8::Isolate* isolate, const gfx::Image& image);
-#if defined(OS_WIN)
-  NativeImage(v8::Isolate* isolate, const base::FilePath& hicon_path);
-#endif
-  ~NativeImage() override;
-
  private:
   v8::Local<v8::Value> ToPNG(gin::Arguments* args);
   v8::Local<v8::Value> ToJPEG(v8::Isolate* isolate, int quality);

shell/common/v8_value_serializer.cc

@@ -8,14 +8,17 @@
 #include <vector>
 
 #include "gin/converter.h"
+#include "shell/common/api/electron_api_native_image.h"
 #include "shell/common/gin_helper/microtasks_scope.h"
+#include "skia/public/mojom/bitmap.mojom.h"
 #include "third_party/blink/public/common/messaging/cloneable_message.h"
+#include "ui/gfx/image/image_skia.h"
 #include "v8/include/v8.h"
 
 namespace electron {
 
 namespace {
-const uint8_t kVersionTag = 0xFF;
+enum SerializationTag { kNativeImageTag = 'i', kVersionTag = 0xFF };
 }  // namespace
 
 class V8Serializer : public v8::ValueSerializer::Delegate {
@@ -62,12 +65,35 @@ class V8Serializer : public v8::ValueSerializer::Delegate {
     data_ = {};
   }
 
+  v8::Maybe<bool> WriteHostObject(v8::Isolate* isolate,
+                                  v8::Local<v8::Object> object) override {
+    api::NativeImage* native_image;
+    if (gin::ConvertFromV8(isolate, object, &native_image)) {
+      // Serialize the NativeImage
+      WriteTag(kNativeImageTag);
+      gfx::ImageSkia image = native_image->image().AsImageSkia();
+      std::vector<gfx::ImageSkiaRep> image_reps = image.image_reps();
+      serializer_.WriteUint32(image_reps.size());
+      for (const auto& rep : image_reps) {
+        serializer_.WriteDouble(rep.scale());
+        const SkBitmap& bitmap = rep.GetBitmap();
+        std::vector<uint8_t> bytes =
+            skia::mojom::InlineBitmap::Serialize(&bitmap);
+        serializer_.WriteUint32(bytes.size());
+        serializer_.WriteRawBytes(bytes.data(), bytes.size());
+      }
+      return v8::Just(true);
+    } else {
+      return v8::ValueSerializer::Delegate::WriteHostObject(isolate, object);
+    }
+  }
+
   void ThrowDataCloneError(v8::Local<v8::String> message) override {
     isolate_->ThrowException(v8::Exception::Error(message));
   }
 
  private:
-  void WriteTag(uint8_t tag) { serializer_.WriteRawBytes(&tag, 1); }
+  void WriteTag(SerializationTag tag) { serializer_.WriteRawBytes(&tag, 1); }
 
   void WriteBlinkEnvelope(uint32_t blink_version) {
     // Write a dummy blink version envelope for compatibility with
@@ -107,6 +133,20 @@ class V8Deserializer : public v8::ValueDeserializer::Delegate {
     return scope.Escape(value);
   }
 
+  v8::MaybeLocal<v8::Object> ReadHostObject(v8::Isolate* isolate) override {
+    uint8_t tag = 0;
+    if (!ReadTag(&tag))
+      return v8::ValueDeserializer::Delegate::ReadHostObject(isolate);
+    switch (tag) {
+      case kNativeImageTag:
+        if (api::NativeImage* native_image = ReadNativeImage(isolate))
+          return native_image->GetWrapper(isolate);
+        break;
+    }
+    // Throws an exception.
+    return v8::ValueDeserializer::Delegate::ReadHostObject(isolate);
+  }
+
  private:
   bool ReadTag(uint8_t* tag) {
     const void* tag_bytes = nullptr;
@@ -127,6 +167,31 @@ class V8Deserializer : public v8::ValueDeserializer::Delegate {
     return true;
   }
 
+  api::NativeImage* ReadNativeImage(v8::Isolate* isolate) {
+    gfx::ImageSkia image_skia;
+    uint32_t num_reps = 0;
+    if (!deserializer_.ReadUint32(&num_reps))
+      return nullptr;
+    for (uint32_t i = 0; i < num_reps; i++) {
+      double scale = 0.0;
+      if (!deserializer_.ReadDouble(&scale))
+        return nullptr;
+      uint32_t bitmap_size_bytes = 0;
+      if (!deserializer_.ReadUint32(&bitmap_size_bytes))
+        return nullptr;
+      const void* bitmap_data = nullptr;
+      if (!deserializer_.ReadRawBytes(bitmap_size_bytes, &bitmap_data))
+        return nullptr;
+      SkBitmap bitmap;
+      if (!skia::mojom::InlineBitmap::Deserialize(bitmap_data,
+                                                  bitmap_size_bytes, &bitmap))
+        return nullptr;
+      image_skia.AddRepresentation(gfx::ImageSkiaRep(bitmap, scale));
+    }
+    gfx::Image image(image_skia);
+    return new api::NativeImage(isolate, image);
+  }
+
   v8::Isolate* isolate_;
   v8::ValueDeserializer deserializer_;
 };

shell/renderer/renderer_client_base.cc

@@ -524,7 +524,6 @@ void RendererClientBase::SetupMainWorldOverrides(
   isolated_api.SetMethod("allowGuestViewElementDefinition",
                          &AllowGuestViewElementDefinition);
   isolated_api.SetMethod("setIsWebView", &SetIsWebView);
-  isolated_api.SetMethod("createNativeImage", &api::NativeImage::CreateEmpty);
 
   auto source_context = GetContext(render_frame->GetWebFrame(), isolate);
   gin_helper::Dictionary global(isolate, source_context->Global());

typings/internal-ambient.d.ts

@@ -6,7 +6,6 @@ declare var isolatedApi: {
   guestViewInternal: any;
   allowGuestViewElementDefinition: NodeJS.InternalWebFrame['allowGuestViewElementDefinition'];
   setIsWebView: (iframe: HTMLIFrameElement) => void;
-  createNativeImage: typeof Electron.nativeImage['createEmpty'];
 }
 
 declare const BUILDFLAG: (flag: boolean) => boolean;