mirror of
https://github.com/nodejs/node.git
synced 2026-10-28 10:15:07 +08:00
Philipp Dunkel
df2ea49e00
Add zlib.crc32c(data[, value]), which computes the CRC-32C checksum (Castagnoli polynomial, RFC 3720 section 12.1). CRC-32C has better error detection properties than the CRC-32 of zlib.crc32() and is the checksum required by many current protocols, formats and services, among them: - iSCSI header and data digests (RFC 7143) - SCTP packet checksums (RFC 9260) - iWARP MPA framing (RFC 5044) - NVMe/TCP header and data digests - ext4, Btrfs and XFS metadata checksums - the Snappy framing format - Apache Kafka record batches - LevelDB and RocksDB block checksums - TensorFlow TFRecord files - Google Cloud Storage and Amazon S3 object checksums Implementing any of these in JavaScript currently needs a userland CRC-32C, which is far slower than native code. The function mirrors zlib.crc32(): it accepts a string (encoded as UTF-8) or an ArrayBufferView, takes an optional 32-bit unsigned starting value so a checksum can be continued across chunks, and has a V8 fast API path. The checksum is computed by absl::ExtendCrc32c() from the bundled Abseil, which is already built and linked into node, so this adds no new dependency and no new checksum implementation. absl_crc32c is added to the libraries linked with --shared-abseil. Abseil selects its hardware-accelerated CRC-32C path (SSE4.2 and PCLMULQDQ on x64, the CRC32 and PMULL extensions on arm64) at compile time, so with node's default compiler flags the portable table-driven implementation is used. It runs at about the same speed as zlib.crc32() (about 6.5 GB/s on a recent x64 machine), while the hardware path reaches about 38 GB/s. Builds whose target CPU baseline already includes those instructions get the hardware path without changes. Enabling it for generic builds is left out on purpose: on x64, Abseil uses the accelerated code for unrecognized CPUs without a runtime feature check, so compiling it in unconditionally would fault on CPUs without SSE4.2. Signed-off-by: Philipp Dunkel <pip@pipobscure.com> PR-URL: https://github.com/nodejs/node/pull/66828 Reviewed-By: Matteo Collina <matteo.collina@gmail.com> Reviewed-By: Paolo Insogna <paolo@cowtech.it> Reviewed-By: Trivikram Kamat <trivikr.dev@gmail.com>
93 lines
3.1 KiB
JavaScript
93 lines
3.1 KiB
JavaScript
'use strict';
|
|
|
|
require('../common');
|
|
const zlib = require('zlib');
|
|
const assert = require('assert');
|
|
const { Buffer } = require('buffer');
|
|
|
|
// Test vectors from RFC 3720, Appendix B.4.
|
|
{
|
|
const ascending = Buffer.alloc(32);
|
|
const descending = Buffer.alloc(32);
|
|
for (let i = 0; i < 32; i++) {
|
|
ascending[i] = i;
|
|
descending[i] = 31 - i;
|
|
}
|
|
const iscsiReadPdu = Buffer.from(
|
|
'01c00000000000000000000000000000' +
|
|
'14000000000004000000001400000018' +
|
|
'28000000000000000200000000000000', 'hex');
|
|
|
|
const tests = [
|
|
[Buffer.alloc(32, 0x00), 0x8a9136aa],
|
|
[Buffer.alloc(32, 0xff), 0x62a8ab43],
|
|
[ascending, 0x46dd794e],
|
|
[descending, 0x113fdb5c],
|
|
[iscsiReadPdu, 0xd9963a56],
|
|
];
|
|
for (const [data, expected] of tests) {
|
|
assert.strictEqual(zlib.crc32c(data), expected);
|
|
assert.strictEqual(zlib.crc32c(data, 0), expected);
|
|
}
|
|
}
|
|
|
|
assert.strictEqual(zlib.crc32c('123456789'), 0xe3069283);
|
|
assert.strictEqual(zlib.crc32c(Buffer.from('123456789')), 0xe3069283);
|
|
|
|
assert.strictEqual(zlib.crc32c(''), 0);
|
|
assert.strictEqual(zlib.crc32c(Buffer.alloc(0)), 0);
|
|
assert.strictEqual(zlib.crc32c('', 0xffffffff), 0xffffffff);
|
|
assert.strictEqual(zlib.crc32c(Buffer.alloc(0), 0x12345678), 0x12345678);
|
|
assert.strictEqual(zlib.crc32c('123456789', -0), 0xe3069283);
|
|
|
|
// Strings are encoded as UTF-8.
|
|
{
|
|
const str = 'héllo wörld ✓ 🚀';
|
|
assert.strictEqual(zlib.crc32c(str), zlib.crc32c(Buffer.from(str, 'utf8')));
|
|
}
|
|
|
|
// Only the bytes covered by the view are used.
|
|
{
|
|
const backing = Buffer.from('xx123456789yy');
|
|
const view = backing.subarray(2, 11);
|
|
assert.strictEqual(zlib.crc32c(view), 0xe3069283);
|
|
assert.strictEqual(
|
|
zlib.crc32c(new Uint8Array(backing.buffer, backing.byteOffset + 2, 9)),
|
|
0xe3069283);
|
|
assert.strictEqual(
|
|
zlib.crc32c(new DataView(backing.buffer, backing.byteOffset + 2, 9)),
|
|
0xe3069283);
|
|
}
|
|
|
|
// Chaining with `value` matches a single pass, across sizes and split points
|
|
// large enough to reach every code path of the implementation.
|
|
{
|
|
const data = Buffer.allocUnsafe(1 << 20);
|
|
for (let i = 0; i < data.length; i++) data[i] = (i * 1103515245 + 12345) & 0xff;
|
|
for (const len of [1, 7, 15, 16, 17, 63, 64, 65, 255, 256, 4095, 4096,
|
|
65537, data.length]) {
|
|
const chunk = data.subarray(0, len);
|
|
const whole = zlib.crc32c(chunk);
|
|
for (const split of new Set([0, 1, len >> 1, len - 1, len])) {
|
|
const crc = zlib.crc32c(chunk.subarray(split),
|
|
zlib.crc32c(chunk.subarray(0, split)));
|
|
assert.strictEqual(crc, whole, `len=${len} split=${split}`);
|
|
}
|
|
}
|
|
}
|
|
|
|
// Different from CRC-32.
|
|
assert.notStrictEqual(zlib.crc32c('123456789'), zlib.crc32('123456789'));
|
|
|
|
[undefined, null, true, 1, () => {}, {}].forEach((invalid) => {
|
|
assert.throws(() => { zlib.crc32c(invalid); }, { code: 'ERR_INVALID_ARG_TYPE' });
|
|
});
|
|
|
|
[null, true, () => {}, {}].forEach((invalid) => {
|
|
assert.throws(() => { zlib.crc32c('test', invalid); }, { code: 'ERR_INVALID_ARG_TYPE' });
|
|
});
|
|
|
|
[-1, 2 ** 32, 1.5, NaN].forEach((invalid) => {
|
|
assert.throws(() => { zlib.crc32c('test', invalid); }, { code: 'ERR_OUT_OF_RANGE' });
|
|
});
|