From fbeb1b3fdc583d3f1aa54c37ef4112fc15f68ebd Mon Sep 17 00:00:00 2001 From: Philipp Dunkel Date: Sun, 18 Oct 2026 13:10:46 +0200 Subject: [PATCH] zlib: add crc32c Add zlib.crc32c(data[, value]), which computes the CRC-32C checksum (Castagnoli polynomial, RFC 3720 section 12.1). CRC-32C has better error detection properties than the CRC-32 of zlib.crc32() and is the checksum required by many current protocols, formats and services, among them: - iSCSI header and data digests (RFC 7143) - SCTP packet checksums (RFC 9260) - iWARP MPA framing (RFC 5044) - NVMe/TCP header and data digests - ext4, Btrfs and XFS metadata checksums - the Snappy framing format - Apache Kafka record batches - LevelDB and RocksDB block checksums - TensorFlow TFRecord files - Google Cloud Storage and Amazon S3 object checksums Implementing any of these in JavaScript currently needs a userland CRC-32C, which is far slower than native code. The function mirrors zlib.crc32(): it accepts a string (encoded as UTF-8) or an ArrayBufferView, takes an optional 32-bit unsigned starting value so a checksum can be continued across chunks, and has a V8 fast API path. The checksum is computed by absl::ExtendCrc32c() from the bundled Abseil, which is already built and linked into node, so this adds no new dependency and no new checksum implementation. absl_crc32c is added to the libraries linked with --shared-abseil. Abseil selects its hardware-accelerated CRC-32C path (SSE4.2 and PCLMULQDQ on x64, the CRC32 and PMULL extensions on arm64) at compile time, so with node's default compiler flags the portable table-driven implementation is used. It runs at about the same speed as zlib.crc32() (about 6.5 GB/s on a recent x64 machine), while the hardware path reaches about 38 GB/s. Builds whose target CPU baseline already includes those instructions get the hardware path without changes. Enabling it for generic builds is left out on purpose: on x64, Abseil uses the accelerated code for unrecognized CPUs without a runtime feature check, so compiling it in unconditionally would fault on CPUs without SSE4.2. Signed-off-by: Philipp Dunkel --- benchmark/zlib/crc32c.js | 47 ++++++++++ configure.py | 1 + doc/api/zlib.md | 45 ++++++++++ lib/zlib.js | 13 ++- src/node_zlib.cc | 34 ++++++++ test/parallel/test-zlib-crc32c.js | 92 ++++++++++++++++++++ test/sequential/test-zlib-crc32c-fast-api.js | 26 ++++++ typings/internalBinding/zlib.d.ts | 1 + 8 files changed, 258 insertions(+), 1 deletion(-) create mode 100644 benchmark/zlib/crc32c.js create mode 100644 test/parallel/test-zlib-crc32c.js create mode 100644 test/sequential/test-zlib-crc32c-fast-api.js diff --git a/benchmark/zlib/crc32c.js b/benchmark/zlib/crc32c.js new file mode 100644 index 000000000000..3a8abd0a286e --- /dev/null +++ b/benchmark/zlib/crc32c.js @@ -0,0 +1,47 @@ +'use strict'; + +const common = require('../common.js'); +const { crc32c } = require('zlib'); + +// Benchmark crc32c on Buffer and String inputs across sizes. +// Iteration count is scaled inversely with input length to keep runtime sane. +// Example: +// node benchmark/zlib/crc32c.js type=buffer len=4096 n=4000000 +// ./out/Release/node benchmark/zlib/crc32c.js --test + +const bench = common.createBenchmark(main, { + type: ['buffer', 'string'], + len: [32, 256, 4096, 65536], + n: [4e6], +}); + +function makeBuffer(size) { + const buf = Buffer.allocUnsafe(size); + for (let i = 0; i < size; i++) buf[i] = (i * 1103515245 + 12345) & 0xff; + return buf; +} + +function makeAsciiString(size) { + const chars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_'; + let out = ''; + for (let i = 0, j = 0; i < size; i++, j = (j + 7) % chars.length) out += chars[j]; + return out; +} + +function main({ type, len, n }) { + // Scale iterations so that total processed bytes roughly constant around n*4096 bytes. + const scale = 4096 / len; + const iters = Math.max(1, Math.floor(n * scale)); + + const data = type === 'buffer' ? makeBuffer(len) : makeAsciiString(len); + + let acc = 0; + for (let i = 0; i < Math.min(iters, 10000); i++) acc ^= crc32c(data, 0); + + bench.start(); + let sum = 0; + for (let i = 0; i < iters; i++) sum ^= crc32c(data, 0); + bench.end(iters); + + if (sum === acc - 1) process.stderr.write(''); +} diff --git a/configure.py b/configure.py index fb742692f530..9c5317e1b7ed 100755 --- a/configure.py +++ b/configure.py @@ -2911,6 +2911,7 @@ def make_bin_override(): 'absl_cleanup', 'absl_config', 'absl_cord', + 'absl_crc32c', 'absl_core_headers', 'absl_die_if_null', 'absl_dynamic_annotations', diff --git a/doc/api/zlib.md b/doc/api/zlib.md index 44caec51b3eb..d65cec6b33a1 100644 --- a/doc/api/zlib.md +++ b/doc/api/zlib.md @@ -2375,6 +2375,49 @@ crc = zlib.crc32(Buffer.from('hello', 'utf16le')); // 1427272415 crc = zlib.crc32(Buffer.from('world', 'utf16le'), crc); // 4150509955 ``` +## `zlib.crc32c(data[, value])` + + + +* `data` {string|Buffer|TypedArray|DataView} When `data` is a string, + it will be encoded as UTF-8 before being used for computation. +* `value` {integer} An optional starting value. It must be a 32-bit unsigned + integer. **Default:** `0` +* Returns: {integer} A 32-bit unsigned integer containing the checksum. + +Computes a 32-bit [Cyclic Redundancy Check][] checksum of `data` using the +Castagnoli polynomial (CRC-32C, [RFC 3720][] section 12.1). If `value` is +specified, it is used as the starting value of the checksum, otherwise, 0 is +used as the starting value. Passing the result of a previous call as `value` +continues the checksum, as with [`zlib.crc32()`][]. + +CRC-32C is a different checksum from the CRC-32 computed by +[`zlib.crc32()`][]; the two produce different results for the same input. +CRC-32C is used by protocols and formats such as iSCSI, SCTP, ext4, Btrfs, +and the Snappy framing format, and has better error detection properties than +CRC-32. + +The CRC algorithm is designed to compute checksums and to detect error +in data transmission. It's not suitable for cryptographic authentication. + +Strings are encoded as UTF-8, in the same way as [`zlib.crc32()`][]. + +```mjs +import zlib from 'node:zlib'; + +let crc = zlib.crc32c('hello'); // 2591144780 +crc = zlib.crc32c('world', crc); // 1456190592 +``` + +```cjs +const zlib = require('node:zlib'); + +let crc = zlib.crc32c('hello'); // 2591144780 +crc = zlib.crc32c('world', crc); // 1456190592 +``` + ## `zlib.createBrotliCompress([options])`