mirror of
https://github.com/ruby/ruby.git
synced 2026-10-28 08:25:20 +08:00
hash.c: speedup ar_find_entry_hint with SWAR
Instead of iterating over the bytes of the ar_table hints
one by one, we can use a little bit of SWAR code to match them
all at once and return the first matching byte index.
In theory we could even re-use the mask to extract all matching
position, however we can't do that because since `ar_equal` calls
back into Ruby, the hints may change after every iteration.
```
compare-ruby: ruby 4.1.0dev (2026-08-06T14:05:57Z master 9658933451) +YJIT +PRISM [arm64-darwin25]
built-ruby: ruby 4.1.0dev (2026-08-06T16:42:50Z ar-find-entry-swar-2 723ff01514) +YJIT +PRISM [arm64-darwin25]
```
| |compare-ruby|built-ruby|
|:-----------|-----------:|---------:|
|hit_first | 13.891M| 15.085M|
| | -| 1.09x|
|hit_third | 13.322M| 15.065M|
| | -| 1.13x|
|hit_fourth | 12.759M| 14.410M|
| | -| 1.13x|
|hit_last | 11.545M| 15.275M|
| | -| 1.32x|
|miss | 10.872M| 16.286M|
| | -| 1.50x|
This commit is contained in:
1 parent
74299e5a87
commit
7da70f1773
3 files changed
+84
-33
Notes:
git
2026-08-14 09:45:45 +00:00
No files matched your search
@@ -0,0 +1,17 @@
|
||||
prelude: |
|
||||
h = {a:1, b:2, c:3, d:4, e:5, f:6, g:7, h:8}
|
||||
e = {}
|
||||
|
||||
benchmark:
|
||||
miss_empty: |
|
||||
e[:a]; e[:a]; e[:a]; e[:a]; e[:a]; e[:a]; e[:a]; e[:a]; e[:a]; e[:a]
|
||||
hit_first: |
|
||||
h[:a]; h[:a]; h[:a]; h[:a]; h[:a]; h[:a]; h[:a]; h[:a]; h[:a]; h[:a]
|
||||
hit_third: |
|
||||
h[:c]; h[:c]; h[:c]; h[:c]; h[:c]; h[:c]; h[:c]; h[:c]; h[:c]; h[:c]
|
||||
hit_fourth: |
|
||||
h[:d]; h[:d]; h[:d]; h[:d]; h[:d]; h[:d]; h[:d]; h[:d]; h[:d]; h[:d]
|
||||
hit_last: |
|
||||
h[:h]; h[:h]; h[:h]; h[:h]; h[:h]; h[:h]; h[:h]; h[:h]; h[:h]; h[:h]
|
||||
miss: |
|
||||
h[:_]; h[:_]; h[:_]; h[:_]; h[:_]; h[:_]; h[:_]; h[:_]; h[:_]; h[:_]
|
||||
@@ -607,49 +607,83 @@ ar_equal(VALUE x, VALUE y)
|
||||
return rb_any_cmp(x, y) == 0;
|
||||
}
|
||||
|
||||
|
||||
#if SIZEOF_VALUE == 8
|
||||
#define AR_HINT_BASE_MASK 0x101010101010101
|
||||
#define AR_HINT_NORMALIZE_MASK 0x7F7F7F7F7F7F7F7F
|
||||
#ifdef WORDS_BIGENDIAN
|
||||
#define AR_HINT_FIND_FIRST_ZERO_BYTE(x) (nlz_int64(x) / CHAR_BIT)
|
||||
#else
|
||||
#define AR_HINT_FIND_FIRST_ZERO_BYTE(x) (ntz_int64(x) / CHAR_BIT)
|
||||
#endif
|
||||
#else
|
||||
#define AR_HINT_BASE_MASK 0x1010101
|
||||
#define AR_HINT_NORMALIZE_MASK 0x7F7F7F7F
|
||||
#ifdef WORDS_BIGENDIAN
|
||||
#define AR_HINT_FIND_FIRST_ZERO_BYTE(x) (nlz_int32(x) / CHAR_BIT)
|
||||
#else
|
||||
#define AR_HINT_FIND_FIRST_ZERO_BYTE(x) (ntz_int32(x) / CHAR_BIT)
|
||||
#endif
|
||||
#endif
|
||||
|
||||
static inline unsigned int
|
||||
ar_hint_first_match(ar_hint_t needle, VALUE haystack)
|
||||
{
|
||||
// Common SWAR technique.
|
||||
// First XOR all bytes so that matching ones are set to 0x00.
|
||||
VALUE search_mask = AR_HINT_BASE_MASK * needle;
|
||||
VALUE matches = haystack ^ search_mask;
|
||||
|
||||
// Then turns 0x00 into 0x80, and any other bytes into 0x00.
|
||||
matches = ~((((matches & AR_HINT_NORMALIZE_MASK) + AR_HINT_NORMALIZE_MASK) | matches) | AR_HINT_NORMALIZE_MASK);
|
||||
unsigned index = AR_HINT_FIND_FIRST_ZERO_BYTE(matches);
|
||||
RBIMPL_ASSERT_OR_ASSUME(index <= RHASH_AR_TABLE_MAX_SIZE);
|
||||
return index;
|
||||
}
|
||||
|
||||
// Returns the bin index if found, RHASH_AR_TABLE_MAX_BOUND if not found,
|
||||
// or RHASH_AR_TABLE_CONVERTED_TO_ST_TABLE if #eql? or a Thread converted the hash to st_table.
|
||||
static unsigned
|
||||
ar_find_entry_hint(VALUE hash, ar_hint_t hint, st_data_t key)
|
||||
{
|
||||
/* if table is NULL, then bound also should be 0 */
|
||||
unsigned first_match = ar_hint_first_match(hint, RHASH_AR_TABLE(hash)->ar_hint.word);
|
||||
|
||||
for (unsigned i = 0; i < RHASH_AR_TABLE_BOUND(hash); i++) {
|
||||
const ar_hint_t *hints = RHASH_AR_TABLE(hash)->ar_hint.ary;
|
||||
if (hints[i] == hint) {
|
||||
ar_table_pair *pair = RHASH_AR_TABLE_REF(hash, i);
|
||||
int eq = ar_equal(key, pair->key);
|
||||
if (UNLIKELY(!RHASH_AR_TABLE_P(hash))) {
|
||||
return RHASH_AR_TABLE_CONVERTED_TO_ST_TABLE;
|
||||
}
|
||||
if (eq) {
|
||||
RB_DEBUG_COUNTER_INC(artable_hint_hit);
|
||||
return i;
|
||||
}
|
||||
else {
|
||||
#if 0
|
||||
static int pid;
|
||||
static char fname[256];
|
||||
static FILE *fp;
|
||||
if (LIKELY(first_match >= RHASH_AR_TABLE_BOUND(hash))) {
|
||||
RB_DEBUG_COUNTER_INC(artable_hint_notfound);
|
||||
return RHASH_AR_TABLE_MAX_BOUND;
|
||||
}
|
||||
|
||||
if (pid != getpid()) {
|
||||
snprintf(fname, sizeof(fname), "/tmp/ruby-armiss.%d", pid = getpid());
|
||||
if ((fp = fopen(fname, "w")) == NULL) rb_bug("fopen");
|
||||
RUBY_ASSERT(RHASH_AR_TABLE(hash)->ar_hint.ary[first_match] == hint);
|
||||
int eq = ar_equal(key, RHASH_AR_TABLE_REF(hash, first_match)->key);
|
||||
if (UNLIKELY(!RHASH_AR_TABLE_P(hash))) {
|
||||
return RHASH_AR_TABLE_CONVERTED_TO_ST_TABLE;
|
||||
}
|
||||
if (LIKELY(eq)) {
|
||||
RB_DEBUG_COUNTER_INC(artable_hint_hit);
|
||||
return first_match;
|
||||
}
|
||||
else {
|
||||
// In theory we could extract all the matching indexes in `ar_hint_first_match`,
|
||||
// and avoid this loop, but sine `ar_equal` may call back into arbitrary code,
|
||||
// the `ar_hint` may have changed.
|
||||
for (unsigned i = first_match + 1; i < RHASH_AR_TABLE_BOUND(hash); i++) {
|
||||
const ar_hint_t *hints = RHASH_AR_TABLE(hash)->ar_hint.ary;
|
||||
if (UNLIKELY(hints[i] == hint)) {
|
||||
eq = ar_equal(key, RHASH_AR_TABLE_REF(hash, i)->key);
|
||||
if (UNLIKELY(!RHASH_AR_TABLE_P(hash))) {
|
||||
return RHASH_AR_TABLE_CONVERTED_TO_ST_TABLE;
|
||||
}
|
||||
if (eq) {
|
||||
RB_DEBUG_COUNTER_INC(artable_hint_hit);
|
||||
return i;
|
||||
}
|
||||
else {
|
||||
RB_DEBUG_COUNTER_INC(artable_hint_miss);
|
||||
}
|
||||
|
||||
st_hash_t h1 = ar_do_hash(key);
|
||||
st_hash_t h2 = ar_do_hash(pair->key);
|
||||
|
||||
fprintf(fp, "miss: hash_eq:%d hints[%d]:%02x hint:%02x\n"
|
||||
" key :%016lx %s\n"
|
||||
" pair->key:%016lx %s\n",
|
||||
h1 == h2, i, hints[i], hint,
|
||||
h1, rb_obj_info(key), h2, rb_obj_info(pair->key));
|
||||
#endif
|
||||
RB_DEBUG_COUNTER_INC(artable_hint_miss);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
RB_DEBUG_COUNTER_INC(artable_hint_notfound);
|
||||
return RHASH_AR_TABLE_MAX_BOUND;
|
||||
}
|
||||
|
||||
+1
-1
@@ -203,7 +203,7 @@ RHASH_AR_TABLE_BOUND(VALUE h)
|
||||
{
|
||||
RUBY_ASSERT(RHASH_AR_TABLE_P(h));
|
||||
const unsigned int bound = RHASH_AR_TABLE_BOUND_RAW(h);
|
||||
RUBY_ASSERT(bound <= RHASH_AR_TABLE_MAX_SIZE);
|
||||
RBIMPL_ASSERT_OR_ASSUME(bound <= RHASH_AR_TABLE_MAX_SIZE);
|
||||
return bound;
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user