zlib: add crc32c

Add zlib.crc32c(data[, value]), which computes the CRC-32C checksum
(Castagnoli polynomial, RFC 3720 section 12.1). CRC-32C has better
error detection properties than the CRC-32 of zlib.crc32() and is the
checksum required by many current protocols, formats and services,
among them:

- iSCSI header and data digests (RFC 7143)
- SCTP packet checksums (RFC 9260)
- iWARP MPA framing (RFC 5044)
- NVMe/TCP header and data digests
- ext4, Btrfs and XFS metadata checksums
- the Snappy framing format
- Apache Kafka record batches
- LevelDB and RocksDB block checksums
- TensorFlow TFRecord files
- Google Cloud Storage and Amazon S3 object checksums

Implementing any of these in JavaScript currently needs a userland
CRC-32C, which is far slower than native code.

The function mirrors zlib.crc32(): it accepts a string (encoded as
UTF-8) or an ArrayBufferView, takes an optional 32-bit unsigned
starting value so a checksum can be continued across chunks, and has a
V8 fast API path.

The checksum is computed by absl::ExtendCrc32c() from the bundled
Abseil, which is already built and linked into node, so this adds no
new dependency and no new checksum implementation. absl_crc32c is
added to the libraries linked with --shared-abseil.

Abseil selects its hardware-accelerated CRC-32C path (SSE4.2 and
PCLMULQDQ on x64, the CRC32 and PMULL extensions on arm64) at compile
time, so with node's default compiler flags the portable table-driven
implementation is used. It runs at about the same speed as zlib.crc32()
(about 6.5 GB/s on a recent x64 machine), while the hardware path
reaches about 38 GB/s. Builds whose target CPU baseline already
includes those instructions get the hardware path without changes.
Enabling it for generic builds is left out on purpose: on x64, Abseil
uses the accelerated code for unrecognized CPUs without a runtime
feature check, so compiling it in unconditionally would fault on CPUs
without SSE4.2.

Signed-off-by: Philipp Dunkel <pip@pipobscure.com>
PR-URL: https://github.com/nodejs/node/pull/66828
Reviewed-By: Matteo Collina <matteo.collina@gmail.com>
Reviewed-By: Paolo Insogna <paolo@cowtech.it>
Reviewed-By: Trivikram Kamat <trivikr.dev@gmail.com>

benchmark/zlib/crc32c.js

@@ -0,0 +1,47 @@
+'use strict';
+
+const common = require('../common.js');
+const { crc32c } = require('zlib');
+
+// Benchmark crc32c on Buffer and String inputs across sizes.
+// Iteration count is scaled inversely with input length to keep runtime sane.
+// Example:
+//   node benchmark/zlib/crc32c.js type=buffer len=4096 n=4000000
+//   ./out/Release/node benchmark/zlib/crc32c.js --test
+
+const bench = common.createBenchmark(main, {
+  type: ['buffer', 'string'],
+  len: [32, 256, 4096, 65536],
+  n: [4e6],
+});
+
+function makeBuffer(size) {
+  const buf = Buffer.allocUnsafe(size);
+  for (let i = 0; i < size; i++) buf[i] = (i * 1103515245 + 12345) & 0xff;
+  return buf;
+}
+
+function makeAsciiString(size) {
+  const chars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_';
+  let out = '';
+  for (let i = 0, j = 0; i < size; i++, j = (j + 7) % chars.length) out += chars[j];
+  return out;
+}
+
+function main({ type, len, n }) {
+  // Scale iterations so that total processed bytes roughly constant around n*4096 bytes.
+  const scale = 4096 / len;
+  const iters = Math.max(1, Math.floor(n * scale));
+
+  const data = type === 'buffer' ? makeBuffer(len) : makeAsciiString(len);
+
+  let acc = 0;
+  for (let i = 0; i < Math.min(iters, 10000); i++) acc ^= crc32c(data, 0);
+
+  bench.start();
+  let sum = 0;
+  for (let i = 0; i < iters; i++) sum ^= crc32c(data, 0);
+  bench.end(iters);
+
+  if (sum === acc - 1) process.stderr.write('');
+}

configure.py

@@ -2923,6 +2923,7 @@ configure_library('abseil', output, pkgname=[
   'absl_cleanup',
   'absl_config',
   'absl_cord',
+  'absl_crc32c',
   'absl_core_headers',
   'absl_die_if_null',
   'absl_dynamic_annotations',

doc/api/zlib.md

@@ -2375,6 +2375,49 @@ crc = zlib.crc32(Buffer.from('hello', 'utf16le'));  // 1427272415
 crc = zlib.crc32(Buffer.from('world', 'utf16le'), crc);  // 4150509955
 ```
 
+## `zlib.crc32c(data[, value])`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+* `data` {string|Buffer|TypedArray|DataView} When `data` is a string,
+  it will be encoded as UTF-8 before being used for computation.
+* `value` {integer} An optional starting value. It must be a 32-bit unsigned
+  integer. **Default:** `0`
+* Returns: {integer} A 32-bit unsigned integer containing the checksum.
+
+Computes a 32-bit [Cyclic Redundancy Check][] checksum of `data` using the
+Castagnoli polynomial (CRC-32C, [RFC 3720][] section 12.1). If `value` is
+specified, it is used as the starting value of the checksum, otherwise, 0 is
+used as the starting value. Passing the result of a previous call as `value`
+continues the checksum, as with [`zlib.crc32()`][].
+
+CRC-32C is a different checksum from the CRC-32 computed by
+[`zlib.crc32()`][]; the two produce different results for the same input.
+CRC-32C is used by protocols and formats such as iSCSI, SCTP, ext4, Btrfs,
+and the Snappy framing format, and has better error detection properties than
+CRC-32.
+
+The CRC algorithm is designed to compute checksums and to detect error
+in data transmission. It's not suitable for cryptographic authentication.
+
+Strings are encoded as UTF-8, in the same way as [`zlib.crc32()`][].
+
+```mjs
+import zlib from 'node:zlib';
+
+let crc = zlib.crc32c('hello');  // 2591144780
+crc = zlib.crc32c('world', crc);  // 1456190592
+```
+
+```cjs
+const zlib = require('node:zlib');
+
+let crc = zlib.crc32c('hello');  // 2591144780
+crc = zlib.crc32c('world', crc);  // 1456190592
+```
+
 ## `zlib.createBrotliCompress([options])`
 
 <!-- YAML
@@ -3382,6 +3425,7 @@ Create a Zstandard decompression transform.
 [Brotli parameters]: #brotli-constants
 [Cyclic redundancy check]: https://en.wikipedia.org/wiki/Cyclic_redundancy_check
 [Memory usage tuning]: #memory-usage-tuning
+[RFC 3720]: https://www.rfc-editor.org/rfc/rfc3720.html
 [RFC 7932]: https://www.rfc-editor.org/rfc/rfc7932.html
 [Streams API]: stream.md
 [Zstd compressor options]: #compressor-options-1
@@ -3451,6 +3495,7 @@ Create a Zstandard decompression transform.
 [`zlib.ZipEntry.createSymlink()`]: #static-method-zlibzipentrycreatesymlinkfilename-target-options
 [`zlib.ZipEntry.createSync()`]: #static-method-zlibzipentrycreatesyncfilename-data-options
 [`zlib.ZipFile.open()`]: #static-method-zlibzipfileopenfilename-options
+[`zlib.crc32()`]: #zlibcrc32data-value
 [`zlib.createZipArchive()`]: #zlibcreateziparchiveentries-options
 [`zlib.createZipArchiveSync()`]: #zlibcreateziparchivesyncentries-options
 [`zlib.getMaxZipContentSize()`]: #zlibgetmaxzipcontentsize

lib/zlib.js

@@ -59,7 +59,7 @@ const {
   isUint8Array,
 } = require('internal/util/types');
 const binding = internalBinding('zlib');
-const { crc32: crc32Native } = binding;
+const { crc32: crc32Native, crc32c: crc32cNative } = binding;
 const assert = require('internal/assert');
 const {
   Buffer,
@@ -1076,6 +1076,16 @@ function crc32(data, value = 0) {
   return crc32Native(data, value);
 }
 
+function crc32c(data, value = 0) {
+  if (typeof data !== 'string' && !isArrayBufferView(data)) {
+    throw new ERR_INVALID_ARG_TYPE('data', ['Buffer', 'TypedArray', 'DataView', 'string'], data);
+  }
+  validateUint32(value, 'value');
+  // Coerce -0 to +0.
+  value += 0;
+  return crc32cNative(data, value);
+}
+
 // Legacy alias on the C++ wrapper object. This is not public API, so we may
 // want to runtime-deprecate it at some point. There's no hurry, though.
 ObjectDefineProperty(binding.Zlib.prototype, 'jsref', {
@@ -1086,6 +1096,7 @@ ObjectDefineProperty(binding.Zlib.prototype, 'jsref', {
 
 module.exports = {
   crc32,
+  crc32c,
   Deflate,
   Inflate,
   Gzip,

src/node_zlib.cc

@@ -34,6 +34,7 @@
 #include "threadpoolwork-inl.h"
 #include "util-inl.h"
 
+#include "absl/crc/crc32c.h"
 #include "node_debug.h"
 #include "v8-fast-api-calls.h"
 #include "v8.h"
@@ -2127,6 +2128,36 @@ static uint32_t FastCRC32(v8::Local<v8::Value> receiver,
 
 static CFunction fast_crc32_(CFunction::Make(FastCRC32));
 
+static inline uint32_t CRC32CImpl(Isolate* isolate,
+                                  Local<Value> data,
+                                  uint32_t value) {
+  return CallOnSequence<uint32_t>(
+      isolate, data, [&](const char* ptr, size_t size) -> uint32_t {
+        return static_cast<uint32_t>(absl::ExtendCrc32c(
+            absl::crc32c_t{value}, std::string_view(ptr, size)));
+      });
+}
+
+static void CRC32C(const FunctionCallbackInfo<Value>& args) {
+  CHECK(args[0]->IsArrayBufferView() || args[0]->IsString());
+  CHECK(args[1]->IsUint32());
+  uint32_t value = args[1].As<v8::Uint32>()->Value();
+  args.GetReturnValue().Set(CRC32CImpl(args.GetIsolate(), args[0], value));
+}
+
+static uint32_t FastCRC32C(v8::Local<v8::Value> receiver,
+                           v8::Local<v8::Value> data,
+                           uint32_t value,
+                           // NOLINTNEXTLINE(runtime/references)
+                           v8::FastApiCallbackOptions& options) {
+  TRACK_V8_FAST_API_CALL("zlib.crc32c");
+  v8::HandleScope handle_scope(options.isolate);
+  CHECK(data->IsArrayBufferView() || data->IsString());
+  return CRC32CImpl(options.isolate, data, value);
+}
+
+static CFunction fast_crc32c_(CFunction::Make(FastCRC32C));
+
 void Initialize(Local<Object> target,
                 Local<Value> unused,
                 Local<Context> context,
@@ -2140,6 +2171,7 @@ void Initialize(Local<Object> target,
   MakeClass<ZstdDecompressStream>::Make(env, target, "ZstdDecompress");
 
   SetFastMethodNoSideEffect(context, target, "crc32", CRC32, &fast_crc32_);
+  SetFastMethodNoSideEffect(context, target, "crc32c", CRC32C, &fast_crc32c_);
   target->Set(env->context(),
               FIXED_ONE_BYTE_STRING(env->isolate(), "ZLIB_VERSION"),
               FIXED_ONE_BYTE_STRING(env->isolate(), ZLIB_VERSION)).Check();
@@ -2153,6 +2185,8 @@ void RegisterExternalReferences(ExternalReferenceRegistry* registry) {
   MakeClass<ZstdDecompressStream>::Make(registry);
   registry->Register(CRC32);
   registry->Register(fast_crc32_);
+  registry->Register(CRC32C);
+  registry->Register(fast_crc32c_);
 }
 
 }  // anonymous namespace

test/parallel/test-zlib-crc32c.js

@@ -0,0 +1,92 @@
+'use strict';
+
+require('../common');
+const zlib = require('zlib');
+const assert = require('assert');
+const { Buffer } = require('buffer');
+
+// Test vectors from RFC 3720, Appendix B.4.
+{
+  const ascending = Buffer.alloc(32);
+  const descending = Buffer.alloc(32);
+  for (let i = 0; i < 32; i++) {
+    ascending[i] = i;
+    descending[i] = 31 - i;
+  }
+  const iscsiReadPdu = Buffer.from(
+    '01c00000000000000000000000000000' +
+    '14000000000004000000001400000018' +
+    '28000000000000000200000000000000', 'hex');
+
+  const tests = [
+    [Buffer.alloc(32, 0x00), 0x8a9136aa],
+    [Buffer.alloc(32, 0xff), 0x62a8ab43],
+    [ascending, 0x46dd794e],
+    [descending, 0x113fdb5c],
+    [iscsiReadPdu, 0xd9963a56],
+  ];
+  for (const [data, expected] of tests) {
+    assert.strictEqual(zlib.crc32c(data), expected);
+    assert.strictEqual(zlib.crc32c(data, 0), expected);
+  }
+}
+
+assert.strictEqual(zlib.crc32c('123456789'), 0xe3069283);
+assert.strictEqual(zlib.crc32c(Buffer.from('123456789')), 0xe3069283);
+
+assert.strictEqual(zlib.crc32c(''), 0);
+assert.strictEqual(zlib.crc32c(Buffer.alloc(0)), 0);
+assert.strictEqual(zlib.crc32c('', 0xffffffff), 0xffffffff);
+assert.strictEqual(zlib.crc32c(Buffer.alloc(0), 0x12345678), 0x12345678);
+assert.strictEqual(zlib.crc32c('123456789', -0), 0xe3069283);
+
+// Strings are encoded as UTF-8.
+{
+  const str = 'héllo wörld ✓ 🚀';
+  assert.strictEqual(zlib.crc32c(str), zlib.crc32c(Buffer.from(str, 'utf8')));
+}
+
+// Only the bytes covered by the view are used.
+{
+  const backing = Buffer.from('xx123456789yy');
+  const view = backing.subarray(2, 11);
+  assert.strictEqual(zlib.crc32c(view), 0xe3069283);
+  assert.strictEqual(
+    zlib.crc32c(new Uint8Array(backing.buffer, backing.byteOffset + 2, 9)),
+    0xe3069283);
+  assert.strictEqual(
+    zlib.crc32c(new DataView(backing.buffer, backing.byteOffset + 2, 9)),
+    0xe3069283);
+}
+
+// Chaining with `value` matches a single pass, across sizes and split points
+// large enough to reach every code path of the implementation.
+{
+  const data = Buffer.allocUnsafe(1 << 20);
+  for (let i = 0; i < data.length; i++) data[i] = (i * 1103515245 + 12345) & 0xff;
+  for (const len of [1, 7, 15, 16, 17, 63, 64, 65, 255, 256, 4095, 4096,
+                     65537, data.length]) {
+    const chunk = data.subarray(0, len);
+    const whole = zlib.crc32c(chunk);
+    for (const split of new Set([0, 1, len >> 1, len - 1, len])) {
+      const crc = zlib.crc32c(chunk.subarray(split),
+                              zlib.crc32c(chunk.subarray(0, split)));
+      assert.strictEqual(crc, whole, `len=${len} split=${split}`);
+    }
+  }
+}
+
+// Different from CRC-32.
+assert.notStrictEqual(zlib.crc32c('123456789'), zlib.crc32('123456789'));
+
+[undefined, null, true, 1, () => {}, {}].forEach((invalid) => {
+  assert.throws(() => { zlib.crc32c(invalid); }, { code: 'ERR_INVALID_ARG_TYPE' });
+});
+
+[null, true, () => {}, {}].forEach((invalid) => {
+  assert.throws(() => { zlib.crc32c('test', invalid); }, { code: 'ERR_INVALID_ARG_TYPE' });
+});
+
+[-1, 2 ** 32, 1.5, NaN].forEach((invalid) => {
+  assert.throws(() => { zlib.crc32c('test', invalid); }, { code: 'ERR_OUT_OF_RANGE' });
+});

test/sequential/test-zlib-crc32c-fast-api.js

@@ -0,0 +1,26 @@
+// Flags: --expose-internals --no-warnings --allow-natives-syntax
+'use strict';
+
+const common = require('../common');
+const assert = require('assert');
+const zlib = require('zlib');
+
+{
+  function testFastPath() {
+    const expected = 0x86a072c0; // zlib.crc32c('test', 0)
+    assert.strictEqual(zlib.crc32c('test', 0), expected);
+    return expected;
+  }
+
+  eval('%PrepareFunctionForOptimization(zlib.crc32c)');
+  testFastPath();
+  eval('%OptimizeFunctionOnNextCall(zlib.crc32c)');
+  testFastPath();
+  testFastPath();
+
+  if (common.isDebug) {
+    const { internalBinding } = require('internal/test/binding');
+    const { getV8FastApiCallCount } = internalBinding('debug');
+    assert.strictEqual(getV8FastApiCallCount('zlib.crc32c'), 2);
+  }
+}

typings/internalBinding/zlib.d.ts

@@ -49,4 +49,5 @@ export interface ZlibBinding {
   ZstdCompress: typeof InternalZlibBinding.ZstdCompress;
   ZstdDecompress: typeof InternalZlibBinding.ZstdDecompress;
   crc32(data: string | ArrayBufferView, value: number): number;
+  crc32c(data: string | ArrayBufferView, value: number): number;
 }