src: add initial support for ESM in embedder API

This patch extends `LoadEnvironment` to support loading ES modules,
and adds the following new types:

```cpp
enum class ModuleFormat : uint8_t {
  kCommonJS,
  kModule,
};

// Data for specifying an entry point script for LoadEnvironment().
// This class uses an opaque layout to allow future additions without
// breaking ABI. Use the setter methods to configure the entry point.
class ModuleData {
  void set_source(std::string_view source);
  void set_format(ModuleFormat format);
  void set_resource_name(std::string_view name);

  std::string_view source() const;
  ModuleFormat format() const;
  std::string_view resource_name() const;
};

class StartExecutionCallbackInfoWithModule {
  void set_env(Environment* env);
  void set_process_object(v8::Local<v8::Object> process_object);
  void set_native_require(v8::Local<v8::Function> native_require);
  void set_run_module(v8::Local<v8::Function> run_module);
  void set_data(void* data);

  Environment* env();
  v8::Local<v8::Object> process();
  v8::Local<v8::Function> native_require();
  v8::Local<v8::Function> run_module();
  void* data();
};
```

And two new `LoadEnvironment()` overloads:

```cpp
// Run entry point with ModuleData configuration
MaybeLocal<Value> LoadEnvironment(
    Environment* env,
    const ModuleData* entry_point,
    EmbedderPreloadCallback preload = nullptr);

// Callback-based with new StartExecutionCallbackInfoWithModule
MaybeLocal<Value> LoadEnvironment(
    Environment* env,
    StartExecutionCallbackWithModule cb,
    EmbedderPreloadCallback preload = nullptr,
    void* callback_data = nullptr);
```

PR-URL: #61548
Refs: #53565
Reviewed-By: Anna Henningsen <[email protected]>
Reviewed-By: Aditi Singh <[email protected]>

Author: joyeecheung

lib/internal/main/embedding.js

@@ -15,15 +15,19 @@ const {
 const { isExperimentalSeaWarningNeeded, isSea } = internalBinding('sea');
 const { emitExperimentalWarning } = require('internal/util');
 const { emitWarningSync } = require('internal/process/warning');
-const { BuiltinModule: { normalizeRequirableId } } = require('internal/bootstrap/realm');
+const { BuiltinModule } = require('internal/bootstrap/realm');
+const { normalizeRequirableId } = BuiltinModule;
 const { Module } = require('internal/modules/cjs/loader');
 const { compileFunctionForCJSLoader } = internalBinding('contextify');
 const { maybeCacheSourceMap } = require('internal/source_map/source_map_cache');
-
 const { codes: {
   ERR_UNKNOWN_BUILTIN_MODULE,
 } } = require('internal/errors');
-
+const { pathToFileURL } = require('internal/url');
+const { loadBuiltinModule } = require('internal/modules/helpers');
+const { moduleFormats } = internalBinding('modules');
+const assert = require('internal/assert');
+const path = require('path');
 // Don't expand process.argv[1] because in a single-executable application or an
 // embedder application, the user main script isn't necessarily provided via the
 // command line (e.g. it could be provided via an API or bundled into the executable).
@@ -44,12 +48,11 @@ if (isExperimentalSeaWarningNeeded()) {
 // value of require.main to module.
 //
 // TODO(RaisinTen): Find a way to deduplicate this.
-function embedderRunCjs(content) {
+function embedderRunCjs(content, filename) {
   // The filename of the module (used for CJS module lookup)
   // is always the same as the location of the executable itself
   // at the time of the loading (which means it changes depending
   // on where the executable is in the file system).
-  const filename = process.execPath;
   const customModule = new Module(filename, null);
 
   const {
@@ -86,33 +89,93 @@ function embedderRunCjs(content) {
   customModule.paths = Module._nodeModulePaths(process.execPath);
   embedderRequire.main = customModule;
 
+  // This currently returns what the wrapper returns i.e. if the code
+  // happens to have a return statement, it returns that; Otherwise it's
+  // undefined.
+  // TODO(joyeecheung): we may want to return the customModule or put it in an
+  // out parameter.
   return compiledWrapper(
     customModule.exports,  // exports
     embedderRequire,       // require
     customModule,          // module
-    process.execPath,      // __filename
+    filename,              // __filename
     customModule.path,     // __dirname
   );
 }
 
 let warnedAboutBuiltins = false;
+function warnNonBuiltinInSEA() {
+  if (isBuiltinWarningNeeded && !warnedAboutBuiltins) {
+    emitWarningSync(
+      'Currently the require() provided to the main script embedded into ' +
+      'single-executable applications only supports loading built-in modules.\n' +
+      'To load a module from disk after the single executable application is ' +
+      'launched, use require("module").createRequire().\n' +
+      'Support for bundled module loading or virtual file systems are under ' +
+      'discussions in https://github.com/nodejs/single-executable');
+    warnedAboutBuiltins = true;
+  }
+}
 
 function embedderRequire(id) {
   const normalizedId = normalizeRequirableId(id);
+
   if (!normalizedId) {
-    if (isBuiltinWarningNeeded && !warnedAboutBuiltins) {
-      emitWarningSync(
-        'Currently the require() provided to the main script embedded into ' +
-        'single-executable applications only supports loading built-in modules.\n' +
-        'To load a module from disk after the single executable application is ' +
-        'launched, use require("module").createRequire().\n' +
-        'Support for bundled module loading or virtual file systems are under ' +
-        'discussions in https://github.com/nodejs/single-executable');
-      warnedAboutBuiltins = true;
-    }
+    warnNonBuiltinInSEA();
     throw new ERR_UNKNOWN_BUILTIN_MODULE(id);
   }
   return require(normalizedId);
 }
 
-return [process, embedderRequire, embedderRunCjs];
+function embedderRunESM(content, filename) {
+  let resourceName;
+  if (path.isAbsolute(filename)) {
+    resourceName = pathToFileURL(filename).href;
+  } else {
+    resourceName = filename;
+  }
+  const { compileSourceTextModule } = require('internal/modules/esm/utils');
+  // TODO(joyeecheung): support code cache, dynamic import() and import.meta.
+  const wrap = compileSourceTextModule(resourceName, content);
+  // Cache the source map for the module if present.
+  if (wrap.sourceMapURL) {
+    maybeCacheSourceMap(resourceName, content, wrap, false, undefined, wrap.sourceMapURL);
+  }
+  const requests = wrap.getModuleRequests();
+  const modules = [];
+  for (let i = 0; i < requests.length; ++i) {
+    const { specifier } = requests[i];
+    const normalizedId = normalizeRequirableId(specifier);
+    if (!normalizedId) {
+      warnNonBuiltinInSEA();
+      throw new ERR_UNKNOWN_BUILTIN_MODULE(specifier);
+    }
+    const mod = loadBuiltinModule(normalizedId);
+    if (!mod) {
+      throw new ERR_UNKNOWN_BUILTIN_MODULE(specifier);
+    }
+    modules.push(mod.getESMFacade());
+  }
+  wrap.link(modules);
+  wrap.instantiate();
+  wrap.evaluate(-1, false);
+
+  // TODO(joyeecheung): we may want to return the v8::Module via a vm.SourceTextModule
+  // when vm.SourceTextModule stablizes, or put it in an out parameter.
+  return wrap.getNamespace();
+}
+
+function embedderRunEntryPoint(content, format, filename) {
+  format ||= moduleFormats.kCommonJS;
+  filename ||= process.execPath;
+
+  if (format === moduleFormats.kCommonJS) {
+    return embedderRunCjs(content, filename);
+  } else if (format === moduleFormats.kModule) {
+    return embedderRunESM(content, filename);
+  }
+  assert.fail(`Unknown format: ${format}`);
+
+}
+
+return [process, embedderRequire, embedderRunEntryPoint];

src/api/environment.cc

@@ -574,7 +574,7 @@ NODE_EXTERN std::unique_ptr<InspectorParentHandle> GetInspectorParentHandle(
 }
 
 MaybeLocal<Value> LoadEnvironment(Environment* env,
-                                  StartExecutionCallback cb,
+                                  StartExecutionCallbackWithModule cb,
                                   EmbedderPreloadCallback preload) {
   env->InitializeLibuv();
   env->InitializeDiagnostics();
@@ -586,22 +586,149 @@ MaybeLocal<Value> LoadEnvironment(Environment* env,
   return StartExecution(env, cb);
 }
 
+struct StartExecutionCallbackInfoWithModule::Impl {
+  Environment* env = nullptr;
+  Local<Object> process_object;
+  Local<Function> native_require;
+  Local<Function> run_module;
+};
+
+StartExecutionCallbackInfoWithModule::StartExecutionCallbackInfoWithModule()
+    : impl_(std::make_unique<Impl>()) {}
+
+StartExecutionCallbackInfoWithModule::~StartExecutionCallbackInfoWithModule() =
+    default;
+
+StartExecutionCallbackInfoWithModule::StartExecutionCallbackInfoWithModule(
+    StartExecutionCallbackInfoWithModule&&) = default;
+
+StartExecutionCallbackInfoWithModule&
+StartExecutionCallbackInfoWithModule::operator=(
+    StartExecutionCallbackInfoWithModule&&) = default;
+
+Environment* StartExecutionCallbackInfoWithModule::env() const {
+  return impl_->env;
+}
+
+Local<Object> StartExecutionCallbackInfoWithModule::process_object() const {
+  return impl_->process_object;
+}
+
+Local<Function> StartExecutionCallbackInfoWithModule::native_require() const {
+  return impl_->native_require;
+}
+
+Local<Function> StartExecutionCallbackInfoWithModule::run_module() const {
+  return impl_->run_module;
+}
+
+void StartExecutionCallbackInfoWithModule::set_env(Environment* env) {
+  impl_->env = env;
+}
+
+void StartExecutionCallbackInfoWithModule::set_process_object(
+    Local<Object> process_object) {
+  impl_->process_object = process_object;
+}
+
+void StartExecutionCallbackInfoWithModule::set_native_require(
+    Local<Function> native_require) {
+  impl_->native_require = native_require;
+}
+
+void StartExecutionCallbackInfoWithModule::set_run_module(
+    Local<Function> run_module) {
+  impl_->run_module = run_module;
+}
+
+struct ModuleData::Impl {
+  std::string_view source;
+  ModuleFormat format = ModuleFormat::kCommonJS;
+  std::string_view resource_name;
+};
+
+ModuleData::ModuleData() : impl_(std::make_unique<Impl>()) {}
+
+ModuleData::~ModuleData() = default;
+
+ModuleData::ModuleData(ModuleData&&) = default;
+
+ModuleData& ModuleData::operator=(ModuleData&&) = default;
+
+void ModuleData::set_source(std::string_view source) {
+  impl_->source = source;
+}
+
+void ModuleData::set_format(ModuleFormat format) {
+  impl_->format = format;
+}
+
+void ModuleData::set_resource_name(std::string_view name) {
+  impl_->resource_name = name;
+}
+
+std::string_view ModuleData::source() const {
+  return impl_->source;
+}
+
+ModuleFormat ModuleData::format() const {
+  return impl_->format;
+}
+
+std::string_view ModuleData::resource_name() const {
+  return impl_->resource_name;
+}
+
+MaybeLocal<Value> LoadEnvironment(Environment* env,
+                                  StartExecutionCallback cb,
+                                  EmbedderPreloadCallback preload) {
+  if (!cb) {
+    return LoadEnvironment(
+        env, StartExecutionCallbackWithModule{}, std::move(preload));
+  }
+
+  return LoadEnvironment(
+      env,
+      [cb = std::move(cb)](const StartExecutionCallbackInfoWithModule& info)
+          -> MaybeLocal<Value> {
+        StartExecutionCallbackInfo legacy_info{
+            info.process_object(), info.native_require(), info.run_module()};
+        return cb(legacy_info);
+      },
+      std::move(preload));
+}
+
 MaybeLocal<Value> LoadEnvironment(Environment* env,
                                   std::string_view main_script_source_utf8,
                                   EmbedderPreloadCallback preload) {
+  ModuleData data;
+  data.set_source(main_script_source_utf8);
+  data.set_format(ModuleFormat::kCommonJS);
+  data.set_resource_name(env->exec_path());
+  return LoadEnvironment(env, &data, std::move(preload));
+}
+
+MaybeLocal<Value> LoadEnvironment(Environment* env,
+                                  const ModuleData* data,
+                                  EmbedderPreloadCallback preload) {
   // It could be empty when it's used by SEA to load an empty script.
-  CHECK_IMPLIES(main_script_source_utf8.size() > 0,
-                main_script_source_utf8.data());
+  CHECK_IMPLIES(data->source().size() > 0, data->source().data());
   return LoadEnvironment(
       env,
-      [&](const StartExecutionCallbackInfo& info) -> MaybeLocal<Value> {
-        Local<Value> main_script;
-        if (!ToV8Value(env->context(), main_script_source_utf8)
-                 .ToLocal(&main_script)) {
-          return {};
-        }
-        return info.run_cjs->Call(
-            env->context(), Null(env->isolate()), 1, &main_script);
+      [data](const StartExecutionCallbackInfoWithModule& info)
+          -> MaybeLocal<Value> {
+        Environment* env = info.env();
+        Local<Context> context = env->context();
+        Isolate* isolate = env->isolate();
+        Local<Value> main_script =
+            ToV8Value(context, data->source()).ToLocalChecked();
+        Local<Value> format =
+            v8::Integer::New(isolate, static_cast<int>(data->format()));
+        Local<Value> resource_name =
+            ToV8Value(context, data->resource_name()).ToLocalChecked();
+        Local<Value> args[] = {main_script, format, resource_name};
+        return info.run_module()->Call(
+            context, Null(isolate), arraysize(args), args);
       },
       std::move(preload));
 }

src/node.cc

@@ -255,37 +255,33 @@ MaybeLocal<Value> StartExecution(Environment* env, const char* main_script_id) {
 }
 
 // Convert the result returned by an intermediate main script into
-// StartExecutionCallbackInfo. Currently the result is an array containing
-// [process, requireFunction, cjsRunner]
-std::optional<StartExecutionCallbackInfo> CallbackInfoFromArray(
-    Local<Context> context, Local<Value> result) {
+// StartExecutionCallbackInfoWithModule. Currently the result is an array
+// containing [process, requireFunction, runModule].
+std::optional<StartExecutionCallbackInfoWithModule> CallbackInfoFromArray(
+    Environment* env, Local<Value> result) {
   CHECK(result->IsArray());
   Local<Array> args = result.As<Array>();
   CHECK_EQ(args->Length(), 3);
-  Local<Value> process_obj, require_fn, runcjs_fn;
+  Local<Value> process_obj, require_fn, run_module;
+  Local<Context> context = env->context();
   if (!args->Get(context, 0).ToLocal(&process_obj) ||
       !args->Get(context, 1).ToLocal(&require_fn) ||
-      !args->Get(context, 2).ToLocal(&runcjs_fn)) {
+      !args->Get(context, 2).ToLocal(&run_module)) {
     return std::nullopt;
   }
   CHECK(process_obj->IsObject());
   CHECK(require_fn->IsFunction());
-  CHECK(runcjs_fn->IsFunction());
-  // TODO(joyeecheung): some support for running ESM as an entrypoint
-  // is needed. The simplest API would be to add a run_esm to
-  // StartExecutionCallbackInfo which compiles, links (to builtins)
-  // and evaluates a SourceTextModule.
-  // TODO(joyeecheung): the env pointer should be part of
-  // StartExecutionCallbackInfo, otherwise embedders are forced to use
-  // lambdas to pass it into the callback, which can make the code
-  // difficult to read.
-  node::StartExecutionCallbackInfo info{process_obj.As<Object>(),
-                                        require_fn.As<Function>(),
-                                        runcjs_fn.As<Function>()};
+  CHECK(run_module->IsFunction());
+  StartExecutionCallbackInfoWithModule info;
+  info.set_env(env);
+  info.set_process_object(process_obj.As<Object>());
+  info.set_native_require(require_fn.As<Function>());
+  info.set_run_module(run_module.As<Function>());
   return info;
 }
 
-MaybeLocal<Value> StartExecution(Environment* env, StartExecutionCallback cb) {
+MaybeLocal<Value> StartExecution(Environment* env,
+                                 StartExecutionCallbackWithModule cb) {
   InternalCallbackScope callback_scope(
       env,
       Object::New(env->isolate()),
@@ -294,7 +290,7 @@ MaybeLocal<Value> StartExecution(Environment* env, StartExecutionCallback cb) {
 
   // Only snapshot builder or embedder applications set the
   // callback.
-  if (cb != nullptr) {
+  if (cb) {
     EscapableHandleScope scope(env->isolate());
 
     Local<Value> result;
@@ -308,9 +304,9 @@ MaybeLocal<Value> StartExecution(Environment* env, StartExecutionCallback cb) {
       }
     }
 
-    auto info = CallbackInfoFromArray(env->context(), result);
+    auto info = CallbackInfoFromArray(env, result);
     if (!info.has_value()) {
-      MaybeLocal<Value>();
+      return MaybeLocal<Value>();
     }
 #if HAVE_INSPECTOR
     if (env->options()->debug_options().break_first_line) {