buffer: add Buffer.stringLength()

Add `Buffer.stringLength(input[, encoding])`, the counterpart of
`Buffer.byteLength()`: it returns the number of UTF-16 code units that
`buf.toString(encoding)` would produce, without decoding.

For UTF-8 the count is computed with simdutf. Invalid input is counted
with the same maximal-subpart replacement that the decoder applies, so
the result always matches `toString().length`. The other encodings are
computed from `byteLength` alone.

This lets code that accumulates streamed input check the result against
`buffer.constants.MAX_STRING_LENGTH` and size its memory budget before
decoding.

Refs: https://github.com/nodejs/node/issues/66062
Signed-off-by: Matteo Collina <hello@matteocollina.com>
PR-URL: https://github.com/nodejs/node/pull/66064
Reviewed-By: Paolo Insogna <paolo@cowtech.it>
Reviewed-By: Gürgün Dayıoğlu <hey@gurgun.day>

benchmark/buffers/buffer-stringlength.js

@@ -0,0 +1,28 @@
+'use strict';
+
+const common = require('../common.js');
+const { Buffer } = require('node:buffer');
+const assert = require('node:assert');
+
+const bench = common.createBenchmark(main, {
+  n: [1e6],
+  encoding: ['utf8', 'latin1', 'base64'],
+  len: [32, 4096, 1048576],
+  input: ['ascii', 'multibyte', 'invalid'],
+});
+
+function main({ n, encoding, len, input }) {
+  let buf;
+  if (input === 'ascii') {
+    buf = Buffer.alloc(len, 'a');
+  } else {
+    buf = Buffer.alloc(len - (len % 3), '€');
+    if (input === 'invalid') buf = Buffer.concat([buf, Buffer.from([0xE2, 0x82])]);
+  }
+  const expected = buf.toString(encoding).length;
+  bench.start();
+  for (let i = 0; i < n; ++i) {
+    assert.strictEqual(Buffer.stringLength(buf, encoding), expected);
+  }
+  bench.end(n);
+}

doc/api/buffer.md

@@ -1015,6 +1015,59 @@ console.log(`${str}: ${str.length} characters, ` +
 When `string` is a {Buffer|DataView|TypedArray|ArrayBuffer|SharedArrayBuffer},
 the byte length as reported by `.byteLength` is returned.
 
+### Static method: `Buffer.stringLength(input[, encoding])`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+* `input` {Buffer | ArrayBuffer | TypedArray} The bytes that would be decoded.
+* `encoding` {string} The character encoding `input` would be decoded with.
+  **Default:** `'utf8'`.
+* Returns: {integer}
+
+Returns the length, in UTF-16 code units, of the string that
+`buf.toString(encoding)` would produce for the same bytes, without decoding
+them. This is the counterpart of [`Buffer.byteLength()`][], which returns the
+number of bytes a string would encode to.
+
+For `'utf8'`, invalid byte sequences are counted as they would be decoded:
+each maximal invalid subsequence becomes one `U+FFFD` replacement character.
+For every other encoding the result is computed from `input.byteLength` alone.
+
+A detached `ArrayBuffer`, or a `TypedArray` backed by one, is treated as empty.
+
+The result is not capped: compare it with
+[`buffer.constants.MAX_STRING_LENGTH`][] before decoding to know whether the
+decode can succeed at all. A string of `n` code units occupies between `n` and
+`2 * n` bytes of memory.
+
+```mjs
+import { Buffer, constants } from 'node:buffer';
+
+const buf = Buffer.from('€ 100', 'utf8');
+
+console.log(Buffer.stringLength(buf));
+// Prints: 5
+console.log(Buffer.stringLength(buf, 'hex'));
+// Prints: 14
+console.log(Buffer.stringLength(buf) <= constants.MAX_STRING_LENGTH);
+// Prints: true
+```
+
+```cjs
+const { Buffer, constants } = require('node:buffer');
+
+const buf = Buffer.from('€ 100', 'utf8');
+
+console.log(Buffer.stringLength(buf));
+// Prints: 5
+console.log(Buffer.stringLength(buf, 'hex'));
+// Prints: 14
+console.log(Buffer.stringLength(buf) <= constants.MAX_STRING_LENGTH);
+// Prints: true
+```
+
 ### Static method: `Buffer.compare(buf1, buf2)`
 
 <!-- YAML
@@ -5726,6 +5779,7 @@ or after startup, if the alignment has to hold at run time.
 [`Buffer.alloc()`]: #static-method-bufferallocsize-fill-encoding
 [`Buffer.allocUnsafe()`]: #static-method-bufferallocunsafesize-alignment
 [`Buffer.allocUnsafeSlow()`]: #static-method-bufferallocunsafeslowsize-alignment
+[`Buffer.byteLength()`]: #static-method-bufferbytelengthstring-encoding
 [`Buffer.concat()`]: #static-method-bufferconcatlist-totallength
 [`Buffer.copyBytesFrom()`]: #static-method-buffercopybytesfromview-offset-length
 [`Buffer.from(array)`]: #static-method-bufferfromarray

lib/buffer.js

@@ -26,6 +26,7 @@ const {
   ArrayBufferIsView,
   ArrayIsArray,
   ArrayPrototypeForEach,
+  MathCeil,
   MathFloor,
   MathMin,
   MathTrunc,
@@ -62,6 +63,7 @@ const {
   fill: bindingFill,
   isAscii: bindingIsAscii,
   isUtf8: bindingIsUtf8,
+  stringLengthUtf8: bindingStringLengthUtf8,
   indexOfBuffer,
   indexOfNumber,
   indexOfString,
@@ -938,6 +940,7 @@ function byteLength(string, encoding) {
 }
 
 Buffer.byteLength = byteLength;
+Buffer.stringLength = stringLength;
 
 // For backwards compatibility.
 ObjectDefineProperty(Buffer.prototype, 'parent', {
@@ -1495,6 +1498,32 @@ function isAscii(input) {
   throw new ERR_INVALID_ARG_TYPE('input', ['ArrayBuffer', 'Buffer', 'TypedArray'], input);
 }
 
+function stringLength(input, encoding = 'utf8') {
+  if (!isTypedArray(input) && !isAnyArrayBuffer(input)) {
+    throw new ERR_INVALID_ARG_TYPE('input', ['ArrayBuffer', 'Buffer', 'TypedArray'], input);
+  }
+  validateString(encoding, 'encoding');
+  const ops = getEncodingOps(encoding);
+  if (ops === undefined) {
+    throw new ERR_UNKNOWN_ENCODING(encoding);
+  }
+  const length = input.byteLength;
+  switch (ops.encodingVal) {
+    case encodingsMap.utf8:
+      return length === 0 ? 0 : bindingStringLengthUtf8(input);
+    case encodingsMap.utf16le:
+      return MathFloor(length / 2);
+    case encodingsMap.hex:
+      return length * 2;
+    case encodingsMap.base64:
+      return MathCeil(length / 3) * 4;
+    case encodingsMap.base64url:
+      return MathCeil(length * 4 / 3);
+    default:  // latin1, ascii
+      return length;
+  }
+}
+
 module.exports = {
   Buffer,
   transcode,

src/node_buffer.cc

@@ -1410,6 +1410,94 @@ static bool FastIsAscii(Local<Value> receiver,
 
 static CFunction fast_is_ascii(CFunction::Make(FastIsAscii));
 
+// Number of UTF-16 code units produced by decoding [p, end) as UTF-8 with
+// WHATWG "maximal subpart" U+FFFD replacement, matching the fallback that
+// StringBytes::Encode takes for invalid input (v8::String::NewFromUtf8).
+static size_t Utf16LengthFromInvalidUtf8(const uint8_t* p, const uint8_t* end) {
+  size_t units = 0;
+  while (p < end) {
+    const uint8_t lead = *p;
+    if (lead < 0x80) {
+      p++;
+      units++;
+      continue;
+    }
+    size_t len;
+    uint8_t lo = 0x80;
+    uint8_t hi = 0xBF;
+    if (lead >= 0xC2 && lead <= 0xDF) {
+      len = 2;
+    } else if (lead >= 0xE0 && lead <= 0xEF) {
+      len = 3;
+      if (lead == 0xE0) lo = 0xA0;
+      if (lead == 0xED) hi = 0x9F;
+    } else if (lead >= 0xF0 && lead <= 0xF4) {
+      len = 4;
+      if (lead == 0xF0) lo = 0x90;
+      if (lead == 0xF4) hi = 0x8F;
+    } else {
+      // Invalid lead byte: one replacement character.
+      p++;
+      units++;
+      continue;
+    }
+    size_t i = 1;
+    for (; i < len && p + i < end; i++) {
+      const uint8_t c = p[i];
+      if (i == 1 ? (c < lo || c > hi) : (c < 0x80 || c > 0xBF)) break;
+    }
+    if (i == len) {
+      p += len;
+      units += (len == 4) ? 2 : 1;
+    } else {
+      // The lead byte plus the valid continuation bytes seen so far form the
+      // maximal subpart and become one replacement character; the byte that
+      // failed is decoded again on the next iteration.
+      p += i;
+      units++;
+    }
+  }
+  return units;
+}
+
+static double StringLengthUtf8Impl(Local<Value> value) {
+  ArrayBufferViewContents<uint8_t> abv(value);
+  const uint8_t* data = abv.data();
+  const size_t length = abv.length();
+  if (length == 0) return 0;
+  const simdutf::result r = simdutf::validate_utf8_with_errors(
+      reinterpret_cast<const char*>(data), length);
+  if (r.error == simdutf::error_code::SUCCESS) {
+    return static_cast<double>(simdutf::utf16_length_from_utf8(
+        reinterpret_cast<const char*>(data), length));
+  }
+  // r.count is the offset of the first invalid sequence; everything before it
+  // is valid UTF-8.
+  const size_t valid = simdutf::utf16_length_from_utf8(
+      reinterpret_cast<const char*>(data), r.count);
+  return static_cast<double>(
+      valid + Utf16LengthFromInvalidUtf8(data + r.count, data + length));
+}
+
+static void StringLengthUtf8(const FunctionCallbackInfo<Value>& args) {
+  CHECK_EQ(args.Length(), 1);
+  CHECK(args[0]->IsTypedArray() || args[0]->IsArrayBuffer() ||
+        args[0]->IsSharedArrayBuffer());
+
+  args.GetReturnValue().Set(StringLengthUtf8Impl(args[0]));
+}
+
+static double FastStringLengthUtf8(Local<Value> receiver,
+                                   Local<Value> value,
+                                   // NOLINTNEXTLINE(runtime/references)
+                                   FastApiCallbackOptions& options) {
+  TRACK_V8_FAST_API_CALL("buffer.stringLengthUtf8");
+  HandleScope scope(options.isolate);
+  return StringLengthUtf8Impl(value);
+}
+
+static CFunction fast_string_length_utf8(CFunction::Make(FastStringLengthUtf8));
+
 void SetBufferPrototype(const FunctionCallbackInfo<Value>& args) {
   Realm* realm = Realm::GetCurrent(args);
 
@@ -1836,6 +1924,11 @@ void Initialize(Local<Object> target,
   SetFastMethodNoSideEffect(context, target, "isUtf8", IsUtf8, &fast_is_utf8);
   SetFastMethodNoSideEffect(
       context, target, "isAscii", IsAscii, &fast_is_ascii);
+  SetFastMethodNoSideEffect(context,
+                            target,
+                            "stringLengthUtf8",
+                            StringLengthUtf8,
+                            &fast_string_length_utf8);
 
   target
       ->Set(context,
@@ -1911,6 +2004,8 @@ void RegisterExternalReferences(ExternalReferenceRegistry* registry) {
   registry->Register(fast_is_utf8);
   registry->Register(IsAscii);
   registry->Register(fast_is_ascii);
+  registry->Register(StringLengthUtf8);
+  registry->Register(fast_string_length_utf8);
 
   registry->Register(StringSlice<ASCII>);
   registry->Register(StringSlice<BASE64>);

test/parallel/test-buffer-stringlength.js

@@ -0,0 +1,134 @@
+'use strict';
+
+require('../common');
+const assert = require('assert');
+const { Buffer } = require('buffer');
+
+const encodings = ['utf8', 'utf-8', 'UTF8', 'ucs2', 'utf16le', 'UTF-16LE',
+                   'latin1', 'binary', 'ascii', 'hex', 'base64', 'base64url'];
+
+function check(buf, encoding = 'utf8') {
+  const expected = buf.toString(encoding).length;
+  assert.strictEqual(Buffer.stringLength(buf, encoding), expected,
+                     `${encoding}: ${buf.toString('hex').slice(0, 64)}`);
+}
+
+// Fixed-width encodings on a range of lengths, including odd ones.
+for (let len = 0; len <= 20; len++) {
+  const buf = Buffer.alloc(len, 0xE9);
+  for (const encoding of encodings) check(buf, encoding);
+}
+
+// Default encoding is utf8.
+assert.strictEqual(Buffer.stringLength(Buffer.from('€ 100')), 5);
+assert.strictEqual(Buffer.stringLength(Buffer.from('😀')), 2);
+assert.strictEqual(Buffer.stringLength(Buffer.alloc(0)), 0);
+
+// Every kind of input that toString-like APIs accept.
+{
+  const source = Buffer.from('héllo wörld €');
+  const ab = new ArrayBuffer(source.length);
+  new Uint8Array(ab).set(source);
+  const sab = new SharedArrayBuffer(source.length);
+  new Uint8Array(sab).set(source);
+  const expected = source.toString().length;
+  assert.strictEqual(Buffer.stringLength(ab), expected);
+  assert.strictEqual(Buffer.stringLength(sab), expected);
+  assert.strictEqual(Buffer.stringLength(new Uint8Array(ab)), expected);
+  assert.strictEqual(Buffer.stringLength(new Uint16Array(ab.slice(0, 16))),
+                     Buffer.from(ab.slice(0, 16)).toString().length);
+}
+
+// Detached ArrayBuffers are empty.
+{
+  const ab = new ArrayBuffer(8);
+  const view = new Uint8Array(ab);
+  structuredClone(ab, { transfer: [ab] });
+  assert.strictEqual(Buffer.stringLength(ab), 0);
+  assert.strictEqual(Buffer.stringLength(view), 0);
+}
+
+// Invalid UTF-8 is counted the way toString() decodes it (one U+FFFD per
+// maximal subpart). Cases taken from test/fixtures/wpt/encoding.
+[
+  [0xFF],
+  [0xC0],
+  [0xE0],
+  [0xC0, 0x00],
+  [0xC0, 0xC0],
+  [0xE0, 0x00],
+  [0xE0, 0xC0],
+  [0xE0, 0x80, 0x00],
+  [0xE0, 0x80, 0xC0],
+  [0xFC, 0x80, 0x80, 0x80, 0x80, 0x80],
+  [0xFE, 0x80, 0x80, 0x80, 0x80, 0x80],
+  [0xC0, 0x80],
+  [0xE0, 0x80, 0x80],
+  [0xF0, 0x80, 0x80, 0x80],
+  [0xF8, 0x80, 0x80, 0x80, 0x80],
+  [0xFC, 0x80, 0x80, 0x80, 0x80, 0x80],
+  [0xC1, 0xBF],
+  [0xE0, 0x81, 0xBF],
+  [0xF0, 0x80, 0x81, 0xBF],
+  [0xF8, 0x80, 0x80, 0x81, 0xBF],
+  [0xFC, 0x80, 0x80, 0x80, 0x81, 0xBF],
+  [0xE0, 0x9F, 0xBF],
+  [0xF0, 0x8F, 0xBF, 0xBF],
+  [0xF8, 0x87, 0xBF, 0xBF, 0xBF],
+  [0xFC, 0x83, 0xBF, 0xBF, 0xBF, 0xBF],
+  [0xED, 0xA0, 0x80],
+  [0xED, 0xBF, 0xBF],
+  [0xF4, 0x90, 0x80, 0x80],
+  [0xF0, 0x9F, 0x98],
+  [0xE2, 0x82],
+  [0xE2, 0x82, 0x41],
+  [0xF0, 0x9F, 0x98, 0x80, 0xF0, 0x9F],
+  [0x41, 0xF0, 0x9F, 0x98, 0x41],
+].forEach((bytes) => {
+  const invalid = Buffer.from(bytes);
+  check(invalid);
+  check(Buffer.concat([Buffer.from('abc'), invalid, Buffer.from('€😀')]));
+  // Push the invalid sequence past the SIMD fast path's prefix.
+  check(Buffer.concat([Buffer.alloc(100, 'é'), invalid, Buffer.alloc(100, 'x')]));
+});
+
+// Random byte soup, checked against the real decoder.
+{
+  let seed = 0x2545F491;
+  const next = () => (seed = (seed * 1103515245 + 12345) >>> 0);
+  for (let i = 0; i < 2000; i++) {
+    const len = next() % 64;
+    const buf = Buffer.alloc(len);
+    for (let j = 0; j < len; j++) {
+      // Bias towards UTF-8 structural bytes so multi-byte prefixes happen.
+      const r = next() % 8;
+      buf[j] = r < 3 ? 0x80 + (next() % 0x40) :
+        r < 5 ? 0xC0 + (next() % 0x40) : next() % 0x100;
+    }
+    check(buf);
+  }
+}
+
+// Large inputs (above the 1 MiB single-pass threshold of the decoder).
+{
+  const big = Buffer.alloc(3 * 2 ** 20, '€');
+  check(big);
+  check(Buffer.concat([big, Buffer.from([0xE2, 0x82])]));
+  check(Buffer.alloc(2 ** 20 + 1, 'a'), 'base64');
+  check(Buffer.alloc(2 ** 20 + 1, 'a'), 'base64url');
+}
+
+// Argument validation.
+for (const input of [undefined, null, 'abc', 42, {}, [], new Blob([]),
+                     new DataView(new ArrayBuffer(4))]) {
+  assert.throws(() => Buffer.stringLength(input),
+                { code: 'ERR_INVALID_ARG_TYPE' });
+}
+for (const encoding of ['nope', 'utf32', '']) {
+  assert.throws(() => Buffer.stringLength(Buffer.alloc(1), encoding),
+                { code: 'ERR_UNKNOWN_ENCODING' });
+}
+for (const encoding of [1, null, {}]) {
+  assert.throws(() => Buffer.stringLength(Buffer.alloc(1), encoding),
+                { code: 'ERR_INVALID_ARG_TYPE' });
+}