mirrors/ruby
0
0 Fork 0
mirror of https://github.com/ruby/ruby.git synced 2026-10-28 16:36:20 +08:00
Code Activity

string.c: Speedup String#tr fastpath with SIMD

Browse Source 
The fast path is now 4.4x faster, now making it ~2.6x times
faster than `CGI.escape_html`, and 26x faster than an equivalent
`gsub`.

```
compare-ruby: ruby 4.1.0dev (2026-09-06T11:22:48Z master 13fc4ffd32) +YJIT +PRISM [arm64-darwin25]
built-ruby: ruby 4.1.0dev (2026-09-06T14:39:43Z string-tr-simd e2526ca9ba) +YJIT +PRISM [arm64-darwin25]
``

|             |compare-ruby|built-ruby|
|:------------|-----------:|---------:|
|tr_escape    |     41.595k|  182.732k|
|             |           -|     4.39x|
|gsub_escape  |      7.135k|    6.977k|
|             |       1.02x|         -|
|cgi_escape   |     70.117k|   70.179k|
|             |           -|     1.00x|

However, contrary to a lookup table, here the performance is linear with
the number of characters to match, hence once we're looking for 16 distinct
bytes, the gain compared to the lookup table is down to ~2x.
This commit is contained in:
Jean Boussier authored and Jean Boussier committed 2026-09-06 20:47:32 +02:00
1 parent e536482403
commit 6acfd9c2b9
3 files changed
+220 -29
Notes: git 2026-09-06 18:48:01 +00:00 
Merged: https://github.com/ruby/ruby/pull/18647
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

benchmark/string_tr.yml
+5 -3
View File
@@ -6,9 +6,11 @@ prelude: |
STR = ((("a" * 31) + "<") * 1000).freeze
ESCAPED_CHARS = {
">" => '\u003e',
"<" => '\u003c',
"&" => '\u0026',
"\\" => "&#39",
"\"" => "&quot;",
">" => '&gt;',
"<" => '&lt;',
"&" => '&amp;',
}.freeze
ESCAPE_PATTERN = Regexp.union(ESCAPED_CHARS.keys)
internal/simd.h
+18
View File
@@ -0,0 +1,18 @@
#ifndef INTERNAL_SIMD_H
#define INTERNAL_SIMD_H
#if defined(__amd64__) || defined(__amd64) || defined(__x86_64__) || defined(__x86_64) || defined(_M_X64) || defined(_M_AMD64)
#ifdef HAVE_X86INTRIN_H
#include <x86intrin.h>
#define HAVE_SIMD 1
#define HAVE_SIMD_SSE2 1
#endif
#endif
#if defined(__ARM_NEON) || defined(__ARM_NEON__) || defined(__aarch64__) || defined(_M_ARM64)
#define HAVE_SIMD 1
#define HAVE_SIMD_NEON 1
#include <arm_neon.h>
#endif
#endif /* INTERNAL_SIMD_H */
string.c
+197 -26
View File
@@ -39,6 +39,7 @@
#include "internal/proc.h"
#include "internal/re.h"
#include "internal/sanitizers.h"
#include "internal/simd.h"
#include "internal/string.h"
#include "internal/transcode.h"
#include "probes.h"
@@ -9211,7 +9212,7 @@ tr_buffer_ensure_capa(struct tr_buffer *buffer, size_t extra_capa)
size_t new_capa = buffer->capa ? buffer->capa : buffer->initial_capa;
RUBY_ASSERT(new_capa >= 32); // Lower would cause infinite loop
while (new_capa < required_capa) {
new_capa *= 1.2;
new_capa = (size_t)(new_capa * 1.2);
}
SIZED_REALLOC_N(buffer->buf, unsigned char, new_capa, buffer->capa);
buffer->ptr = buffer->buf + offset;
@@ -9220,7 +9221,7 @@ tr_buffer_ensure_capa(struct tr_buffer *buffer, size_t extra_capa)
}
static inline void
tr_buffer_append(struct tr_buffer *buffer, unsigned char *ptr, size_t len)
tr_buffer_append(struct tr_buffer *buffer, const unsigned char *ptr, size_t len)
{
if (len) {
tr_buffer_ensure_capa(buffer, len);
@@ -9286,6 +9287,164 @@ tr_trans_pairs_coerce_i(st_data_t key, st_data_t value, st_data_t _args)
return ST_CONTINUE;
}
#define TR_TRANS_PAIRS_SIMD_MAX_NEEDLES 16
struct tr_trans_pairs_search {
const unsigned char *s;
const unsigned char *send;
#ifdef HAVE_SIMD
unsigned char needles[TR_TRANS_PAIRS_SIMD_MAX_NEEDLES];
int needles_count;
#ifdef HAVE_SIMD_NEON
uint64_t matches_bitmap;
#endif
#ifdef HAVE_SIMD_SSE2
int matches_bitmap;
#endif
#endif
VALUE trans_table[256];
};
static inline VALUE
tr_trans_pairs_search_basic(struct tr_trans_pairs_search *search)
{
while (search->s < search->send) {
VALUE repl = search->trans_table[*search->s];
if (UNLIKELY(repl)) {
return repl;
}
search->s++;
}
return 0;
}
#ifdef HAVE_SIMD_SSE2
static inline bool
tr_trans_pairs_next_match_sse2(struct tr_trans_pairs_search *search)
{
size_t next_match_offset = ntz_int32(search->matches_bitmap);
search->matches_bitmap >>= (next_match_offset + 1);
search->s += next_match_offset;
if (search->s > search->send) {
search->s = search->send;
return false;
}
return true;
}
static inline VALUE
tr_trans_pairs_search_sse2(struct tr_trans_pairs_search *search)
{
RBIMPL_ASSERT_OR_ASSUME(search->needles_count > 0);
RBIMPL_ASSERT_OR_ASSUME(search->needles_count < TR_TRANS_PAIRS_SIMD_MAX_NEEDLES);
if (search->matches_bitmap) {
return tr_trans_pairs_next_match_sse2(search);
}
if ((size_t)(search->send - search->s) >= sizeof(__m128i)) {
int i;
__m128i masks[TR_TRANS_PAIRS_SIMD_MAX_NEEDLES];
for (i = 0; i < search->needles_count; i++) {
masks[i] = _mm_set1_epi8(search->needles[i]);
}
do {
const __m128i bytes = _mm_loadu_si128((__m128i const *)search->s);
__m128i matches[TR_TRANS_PAIRS_SIMD_MAX_NEEDLES];
for (i = 0; i < search->needles_count; i++) {
matches[i] = _mm_cmpeq_epi8(bytes, masks[i]);
}
for (i = i; i < search->needles_count; i++) {
matches[0] = _mm_or_si128(matches[0], matches[i]);
}
const int bitmap = _mm_movemask_epi8(matches[0]);
if (bitmap) {
search->matches_bitmap = bitmap;
return tr_trans_pairs_next_match_sse2(search);
}
search->s += sizeof(__m128i);
} while ((size_t)(search->send - search->s) >= sizeof(__m128i));
}
return tr_trans_pairs_search_basic(search);
}
#define tr_trans_pairs_search_impl tr_trans_pairs_search_sse2
#endif
#ifdef HAVE_SIMD_NEON
static inline bool
tr_trans_pairs_next_match_neon(struct tr_trans_pairs_search *search)
{
size_t next_match_offset = ntz_int64(search->matches_bitmap) / 4;
search->matches_bitmap >>= (next_match_offset + 1) * 4;
search->s += next_match_offset;
if (search->s > search->send) {
search->s = search->send;
return false;
}
return true;
}
static inline VALUE
tr_trans_pairs_search_neon(struct tr_trans_pairs_search *search)
{
if (search->needles_count) {
RBIMPL_ASSERT_OR_ASSUME(search->needles_count > 0);
RBIMPL_ASSERT_OR_ASSUME(search->needles_count <= TR_TRANS_PAIRS_SIMD_MAX_NEEDLES);
if (search->matches_bitmap) {
return tr_trans_pairs_next_match_neon(search);
}
if ((size_t)(search->send - search->s) >= sizeof(uint8x16_t)) {
int i;
uint8x16_t masks[TR_TRANS_PAIRS_SIMD_MAX_NEEDLES];
for (i = 0; i < search->needles_count; i++) {
masks[i] = vdupq_n_u8(search->needles[i]);
}
do {
const uint8x16_t bytes = vld1q_u8(search->s);
uint8x16_t matches[TR_TRANS_PAIRS_SIMD_MAX_NEEDLES];
for (i = 0; i < search->needles_count; i++) {
matches[i] = vceqq_u8(bytes, masks[i]);
}
for (i = i; i < search->needles_count; i++) {
matches[0] = vorrq_u8(matches[0], matches[i]);
}
const uint8x8_t res = vshrn_n_u16(vreinterpretq_u16_u8(matches[0]), 4);
const uint64_t bitmap = vget_lane_u64(vreinterpret_u64_u8(res), 0) & 0x8888888888888888ull;
if (bitmap) {
search->matches_bitmap = bitmap;
return tr_trans_pairs_next_match_neon(search);
}
search->s += sizeof(uint8x16_t);
} while ((size_t)(search->send - search->s) >= sizeof(uint8x16_t));
}
}
return tr_trans_pairs_search_basic(search);
}
#define tr_trans_pairs_search_impl tr_trans_pairs_search_neon
#endif
#ifndef tr_trans_pairs_search_impl
#define tr_trans_pairs_search_impl tr_trans_pairs_search_basic
#endif
static VALUE
tr_trans_pairs(VALUE str, VALUE pairs_val)
{
@@ -9311,17 +9470,19 @@ tr_trans_pairs(VALUE str, VALUE pairs_val)
VALUE hash = 0;
unsigned char *sstart = (unsigned char *)RSTRING_PTR(str);
unsigned char *s = sstart;
unsigned char *send = sstart + RSTRING_LEN(str);
const unsigned char *sstart = (unsigned char *)RSTRING_PTR(str);
long str_len = RSTRING_LEN(str);
int termlen = rb_enc_mbminlen(e1);
struct tr_buffer buffer;
tr_buffer_init(&buffer, send - s);
tr_buffer_init(&buffer, str_len);
bool modify = false;
if (RB_LIKELY(rb_str_encindex_fastpath(rb_enc_to_index(e1)))) {
VALUE trans_table[256] = { 0 };
struct tr_trans_pairs_search search = {
.s = sstart,
.send = sstart + str_len,
};
for (size_t index = 0; index < pairs_count; index++) {
struct tr_pair *pair = &pairs[index];
@@ -9329,11 +9490,20 @@ tr_trans_pairs(VALUE str, VALUE pairs_val)
char *ptr = RSTRING_PTR(pair->search);
unsigned int codepoint = rb_enc_mbc_to_codepoint(ptr, RSTRING_END(pair->search), e1);
const unsigned char first_byte = (unsigned char)*ptr;
#ifdef HAVE_SIMD
if (pairs_count <= TR_TRANS_PAIRS_SIMD_MAX_NEEDLES) {
search.needles[index] = first_byte;
search.needles_count++;
}
#endif
if (rb_enc_codelen(codepoint, e1) == 1) {
trans_table[(unsigned char)*ptr] = pair->replace;
search.trans_table[first_byte] = pair->replace;
}
else {
trans_table[(unsigned char)*ptr] = Qundef;
search.trans_table[first_byte] = Qundef;
if (!hash) {
hash = rb_obj_hide(rb_hash_new_capa(pairs_count));
}
@@ -9341,42 +9511,43 @@ tr_trans_pairs(VALUE str, VALUE pairs_val)
}
}
unsigned char *checkpoint = s;
while (s < send) {
VALUE repl = trans_table[*s];
const unsigned char *checkpoint = search.s;
VALUE repl;
while ((repl = tr_trans_pairs_search_impl(&search))) {
int clen = 1;
if (UNLIKELY(repl == Qundef)) {
unsigned int c = rb_enc_mbc_to_codepoint((char *)s, (char *)send, e1);
unsigned int c = rb_enc_mbc_to_codepoint((char *)search.s, (char *)search.send, e1);
clen = rb_enc_codelen(c, e1);
repl = rb_hash_lookup2(hash, UINT2NUM(c), 0);
}
if (LIKELY(repl == 0)) {
s += clen;
continue;
if (!repl) {
search.s += clen;
continue;
}
}
modify = true;
if (checkpoint < s) {
tr_buffer_append(&buffer, checkpoint, s - checkpoint);
if (checkpoint < search.s) {
tr_buffer_append(&buffer, checkpoint, search.s - checkpoint);
}
tr_buffer_append_str(&buffer, repl);
s += clen;
checkpoint = s;
search.s += clen;
checkpoint = search.s;
if (cr == ENC_CODERANGE_7BIT && rb_enc_str_coderange(repl) != ENC_CODERANGE_7BIT) {
cr == ENC_CODERANGE_VALID;
cr = ENC_CODERANGE_VALID;
}
}
if (modify && checkpoint < s) {
tr_buffer_append(&buffer, checkpoint, s - checkpoint);
if (modify && checkpoint < search.s) {
tr_buffer_append(&buffer, checkpoint, search.s - checkpoint);
}
}
else {
const unsigned char *s = sstart;
const unsigned char *send = sstart + str_len;
hash = rb_obj_hide(rb_hash_new_capa(pairs_count));
for (size_t index = 0; index < pairs_count; index++) {