src: do not enable wasm trap handler if there's not enough vmem

PR-URL: https://github.com/nodejs/node/pull/62132
Refs: https://github.com/microsoft/vscode/issues/251777
Refs: https://chromium-review.googlesource.com/c/v8/v8/+/7638233
Reviewed-By: Aditi Singh <aditisingh1400@gmail.com>

doc/api/cli.md

@@ -754,40 +754,36 @@ vm.measureMemory();
 added:
 - v22.2.0
 - v20.15.0
+changes:
+  - version:
+    - REPLACEME
+    pr-url: https://github.com/nodejs/node/pull/62132
+    description: Node.js now automatically disables the trap handler when there is not
+                 enough virtual memory available at startup to allocate one cage.
 -->
 
-By default, Node.js enables trap-handler-based WebAssembly bound
-checks. As a result, V8 does not need to insert inline bound checks
-in the code compiled from WebAssembly which may speed up WebAssembly
-execution significantly, but this optimization requires allocating
-a big virtual memory cage (currently 10GB). If the Node.js process
-does not have access to a large enough virtual memory address space
-due to system configurations or hardware limitations, users won't
-be able to run any WebAssembly that involves allocation in this
-virtual memory cage and will see an out-of-memory error.
+Node.js enables V8's trap-handler-based WebAssembly bound checks on 64-bit platforms,
+which significantly improves WebAssembly performance by eliminating the need for
+inline bound checks. This optimization requires allocating a large virtual memory
+cage per WebAssembly memory instance (currently typically 8GB for 32-bit WebAssembly memory,
+16GB for 64-bit WebAssembly memory) to trap out-of-bound accesses. On most 64-bit
+platforms, the virtual memory address space is usually large enough (around 128TB)
+to accommodate typical WebAssembly usages, but if the machine has manual limits
+on virtual memory (e.g. through `ulimit -v`), WebAssembly memory allocation is
+more likely to fail with `WebAssembly.Memory(): could not allocate memory`.
 
-```console
-$ ulimit -v 5000000
-$ node -p "new WebAssembly.Memory({ initial: 10, maximum: 100 });"
-[eval]:1
-new WebAssembly.Memory({ initial: 10, maximum: 100 });
-^
+At startup, Node.js automatically checks whether there is enough virtual memory
+available to allocate at least one cage, and if not, the trap-handler optimization
+is automatically disabled so that WebAssembly can still run using inline
+bound checks (with less optimal performance). But if the application needs to create
+many WebAssembly memory instances and the machine still configures a relatively high
+limit on virtual memory, allocation of WebAssembly memory instances may still fail
+more quickly than expected due to the raised virtual memory usage.
 
-RangeError: WebAssembly.Memory(): could not allocate memory
-    at [eval]:1:1
-    at runScriptInThisContext (node:internal/vm:209:10)
-    at node:internal/process/execution:118:14
-    at [eval]-wrapper:6:24
-    at runScript (node:internal/process/execution:101:62)
-    at evalScript (node:internal/process/execution:136:3)
-    at node:internal/main/eval_string:49:3
-
-```
-
-`--disable-wasm-trap-handler` disables this optimization so that
-users can at least run WebAssembly (with less optimal performance)
-when the virtual memory address space available to their Node.js
-process is lower than what the V8 WebAssembly memory cage needs.
+`--disable-wasm-trap-handler` fully disables this optimization so that WebAssembly memory
+instances always use inline bound checks instead of reserving large virtual memory cages.
+This allows more instances to be created when the virtual memory address space available
+to the Node.js process is limited.
 
 ### `--disallow-code-generation-from-strings`
 

src/debug_utils.h

@@ -44,6 +44,7 @@ void NODE_EXTERN_PRIVATE FWrite(FILE* file, const std::string& str);
 // from a provider type to a debug category.
 #define DEBUG_CATEGORY_NAMES(V)                                                \
   NODE_ASYNC_PROVIDER_TYPES(V)                                                 \
+  V(BOOTSTRAP)                                                                 \
   V(CRYPTO)                                                                    \
   V(COMPILE_CACHE)                                                             \
   V(DIAGNOSTICS)                                                               \

src/node.cc

@@ -1046,6 +1046,45 @@ static ExitCode InitializeNodeWithArgsInternal(
   return ExitCode::kNoFailure;
 }
 
+#if NODE_USE_V8_WASM_TRAP_HANDLER
+bool CanEnableWebAssemblyTrapHandler() {
+// On POSIX, the machine may have a limit on the amount of virtual memory
+// available, if it's not enough to allocate at least one cage for WASM,
+// then the trap-handler-based bound checks cannot be used.
+#ifdef __POSIX__
+  struct rlimit lim;
+  if (getrlimit(RLIMIT_AS, &lim) != 0 || lim.rlim_cur == RLIM_INFINITY) {
+    // Can't get the limit or there's no limit, assume trap handler can be
+    // enabled.
+    return true;
+  }
+  uint64_t virtual_memory_available = static_cast<uint64_t>(lim.rlim_cur);
+
+  size_t byte_capacity = 64 * 1024;  // 64KB, the minimum size of a WASM memory.
+  uint64_t cage_size_needed_32 = V8::GetWasmMemoryReservationSizeInBytes(
+      V8::WasmMemoryType::kMemory32, byte_capacity);
+  uint64_t cage_size_needed_64 = V8::GetWasmMemoryReservationSizeInBytes(
+      V8::WasmMemoryType::kMemory64, byte_capacity);
+  uint64_t cage_size_needed =
+      std::max(cage_size_needed_32, cage_size_needed_64);
+  bool can_enable = virtual_memory_available >= cage_size_needed;
+  per_process::Debug(DebugCategory::BOOTSTRAP,
+                     "Virtual memory available: %" PRIu64 " bytes,\n"
+                     "cage size needed for 32-bit: %" PRIu64 " bytes,\n"
+                     "cage size needed for 64-bit: %" PRIu64 " bytes,\n"
+                     "Can%senable WASM trap handler\n",
+                     virtual_memory_available,
+                     cage_size_needed_32,
+                     cage_size_needed_64,
+                     can_enable ? " " : " not ");
+
+  return can_enable;
+#else
+  return true;
+#endif  // __POSIX__
+}
+#endif  // NODE_USE_V8_WASM_TRAP_HANDLER
+
 static std::shared_ptr<InitializationResultImpl>
 InitializeOncePerProcessInternal(const std::vector<std::string>& args,
                                  ProcessInitializationFlags::Flags flags =
@@ -1248,7 +1287,9 @@ InitializeOncePerProcessInternal(const std::vector<std::string>& args,
   bool use_wasm_trap_handler =
       !per_process::cli_options->disable_wasm_trap_handler;
   if (!(flags & ProcessInitializationFlags::kNoDefaultSignalHandling) &&
-      use_wasm_trap_handler) {
+      use_wasm_trap_handler && CanEnableWebAssemblyTrapHandler()) {
+    per_process::Debug(DebugCategory::BOOTSTRAP,
+                       "Enabling WebAssembly trap handler for bounds checks\n");
 #if defined(_WIN32)
     constexpr ULONG first = TRUE;
     per_process::old_vectored_exception_handler =

test/testpy/__init__.py

@@ -36,6 +36,7 @@ FLAGS_PATTERN = re.compile(r"//\s+Flags:(.*)")
 LS_RE = re.compile(r'^test-.*\.m?js$')
 ENV_PATTERN = re.compile(r"//\s+Env:(.*)")
 NODE_TEST_PATTERN = re.compile(r"('|`|\")node:test\1")
+RLIMIT_AS_PATTERN = re.compile(r"//\s+RLIMIT_AS:\s*(\d+)")
 
 class SimpleTestCase(test.TestCase):
 
@@ -99,6 +100,10 @@ class SimpleTestCase(test.TestCase):
       else:
         result += flags
 
+    rlimit_as_match = RLIMIT_AS_PATTERN.search(source)
+    if rlimit_as_match:
+      self.max_virtual_memory = int(rlimit_as_match.group(1))
+
     if self.context.use_error_reporter and NODE_TEST_PATTERN.search(source):
       result += ['--test-reporter=./test/common/test-error-reporter.js',
                  '--test-reporter-destination=stdout']
@@ -189,15 +194,3 @@ class AbortTestConfiguration(SimpleTestConfiguration):
     for tst in result:
       tst.disable_core_files = True
     return result
-
-class WasmAllocationTestConfiguration(SimpleTestConfiguration):
-  def __init__(self, context, root, section, additional=None):
-    super(WasmAllocationTestConfiguration, self).__init__(context, root, section,
-                                                          additional)
-
-  def ListTests(self, current_path, path, arch, mode):
-    result = super(WasmAllocationTestConfiguration, self).ListTests(
-         current_path, path, arch, mode)
-    for tst in result:
-      tst.max_virtual_memory = 5 * 1024 * 1024 * 1024 # 5GB
-    return result

test/wasm-allocation/test-wasm-allocation-auto-adapt.js

@@ -0,0 +1,12 @@
+// RLIMIT_AS: 3221225472
+// When the virtual memory limit is 3GB, there is not enough virtual memory for
+// even one wasm cage. In this case Node.js should automatically adapt and
+// skip enabling trap-based bounds checks, so that WASM can at least run with
+// inline bound checks.
+'use strict';
+
+require('../common');
+new WebAssembly.Memory({ initial: 10, maximum: 100 });
+
+// Test memory64 works too.
+new WebAssembly.Memory({ address: 'i64', initial: 10n, maximum: 100n });

test/wasm-allocation/test-wasm-allocation-disable-trap-handler.js

@@ -0,0 +1,20 @@
+// Flags: --disable-wasm-trap-handler
+// RLIMIT_AS: 34359738368
+
+// 32GB should be enough for at least 2 cages, but not enough for 30.
+
+// Test that with limited virtual memory space, --disable-wasm-trap-handler
+// fully disables trap-based bounds checks, and thus allows WASM to run with
+// inline bound checks.
+'use strict';
+
+require('../common');
+const instances = [];
+for (let i = 0; i < 30; i++) {
+  instances.push(new WebAssembly.Memory({ initial: 10, maximum: 100 }));
+}
+
+// Test memory64 works too.
+for (let i = 0; i < 30; i++) {
+  instances.push(new WebAssembly.Memory({ initial: 10n, maximum: 100n, address: 'i64' }));
+}

test/wasm-allocation/test-wasm-allocation-memory64.js

@@ -0,0 +1,22 @@
+// RLIMIT_AS: 21474836480
+// With 20GB virtual memory, there's enough space for the first wasm memory64
+// allocation to succeed, but not enough for subsequent ones since each
+// wasm memory64 with guard regions reserves 16GB of virtual address space.
+'use strict';
+
+require('../common');
+const assert = require('assert');
+
+// The first allocation should succeed.
+const first = new WebAssembly.Memory({ address: 'i64', initial: 10n, maximum: 100n });
+assert(first);
+
+// Subsequent allocations should eventually fail due to running out of
+// virtual address space. memory64 reserves 16GB per allocation (vs 8GB for
+// memory32), so the limit is reached even faster.
+assert.throws(() => {
+  const instances = [first];
+  for (let i = 1; i < 30; i++) {
+    instances.push(new WebAssembly.Memory({ address: 'i64', initial: 10n, maximum: 100n }));
+  }
+}, /WebAssembly\.Memory/);

test/wasm-allocation/test-wasm-allocation.js

@@ -1,7 +1,21 @@
-// Flags: --disable-wasm-trap-handler
-// Test that with limited virtual memory space, --disable-wasm-trap-handler
-// allows WASM to at least run with inline bound checks.
+// RLIMIT_AS: 21474836480
+// With 20GB virtual memory, there's enough space for the first few wasm memory
+// allocation to succeed, but not enough for many subsequent ones since each
+// wasm memory32 with guard regions reserves 8GB of virtual address space.
 'use strict';
 
 require('../common');
-new WebAssembly.Memory({ initial: 10, maximum: 100 });
+const assert = require('assert');
+
+// The first allocation should succeed.
+const first = new WebAssembly.Memory({ initial: 10, maximum: 100 });
+assert(first);
+
+// Subsequent allocations should eventually fail due to running out of
+// virtual address space.
+assert.throws(() => {
+  const instances = [first];
+  for (let i = 1; i < 30; i++) {
+    instances.push(new WebAssembly.Memory({ initial: 10, maximum: 100 }));
+  }
+}, /WebAssembly\.Memory/);

test/wasm-allocation/testcfg.py

@@ -3,4 +3,4 @@ sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
 import testpy
 
 def GetConfiguration(context, root):
-  return testpy.WasmAllocationTestConfiguration(context, root, 'wasm-allocation')
+  return testpy.SimpleTestConfiguration(context, root, 'wasm-allocation')

test/wasm-allocation/wasm-allocation.status

@@ -8,3 +8,9 @@ prefix wasm-allocation
 
 [$system!=linux || $asan==on || $pointer_compression==on]
 test-wasm-allocation: SKIP
+test-wasm-allocation-auto-adapt: SKIP
+test-wasm-allocation-memory64: SKIP
+
+[$arch!=x64 && $arch!=arm64]
+test-wasm-allocation: SKIP
+test-wasm-allocation-memory64: SKIP