Optimize jemalloc fastpaths: emap_alloc_ctx_lookup and sdallocx

- Inline the L1 cache hit in emap_alloc_ctx_lookup() and move the slowpath
  out of line to rtree_read_slow().
- Avoid dereferencing edata for slab allocations in emap_alloc_ctx_lookup().
- Optimize je_sdallocx_impl() by checking flags == 0 for fastpath.

include/jemalloc/internal/emap.h

@@ -261,17 +261,37 @@ emap_alloc_ctx_lookup(
     tsdn_t *tsdn, emap_t *emap, const void *ptr, emap_alloc_ctx_t *alloc_ctx) {
 	EMAP_DECLARE_RTREE_CTX;
 
-	rtree_contents_t contents = rtree_read(
-	    tsdn, &emap->rtree, rtree_ctx, (uintptr_t)ptr);
+	/* Inline the L1 cache hit; everything else is out of line. */
+	rtree_leaf_elm_t *elm;
+	rtree_contents_t  contents;
+	if (likely(!rtree_leaf_elm_lookup_fast(
+	        tsdn, &emap->rtree, rtree_ctx, (uintptr_t)ptr, &elm))) {
+		contents = rtree_leaf_elm_read(
+		    tsdn, &emap->rtree, elm, /* dependent */ true);
+	} else {
+		contents = rtree_read_slow(
+		    tsdn, &emap->rtree, rtree_ctx, (uintptr_t)ptr);
+	}
 	/*
 	 * If the alloc is invalid, do not calculate usize since edata
 	 * could be corrupted.
+	 * For slabs or when large size classes are enabled, usize can be
+	 * computed directly from szind without dereferencing edata.
 	 */
+	size_t usize;
+	if (likely(contents.metadata.slab
+	        || !sz_large_size_classes_disabled())) {
+		usize = (contents.metadata.szind == SC_NSIZES)
+		    ? 0
+		    : sz_index2size(contents.metadata.szind);
+	} else {
+		usize = (contents.metadata.szind == SC_NSIZES
+		            || contents.edata == NULL)
+		    ? 0
+		    : edata_usize_get(contents.edata);
+	}
 	emap_alloc_ctx_init(alloc_ctx, contents.metadata.szind,
-	    contents.metadata.slab,
-	    (contents.metadata.szind == SC_NSIZES || contents.edata == NULL)
-	        ? 0
-	        : edata_usize_get(contents.edata));
+	    contents.metadata.slab, usize);
 }
 
 /* The pointer must be mapped. */

include/jemalloc/internal/jemalloc_internal_inlines_c.h

@@ -568,9 +568,12 @@ je_sdallocx_noflags(void *ptr, size_t size) {
 
 JEMALLOC_ALWAYS_INLINE void JEMALLOC_NOTHROW
 je_sdallocx_impl(void *ptr, size_t size, int flags) {
-	if (flags != 0 || !free_fastpath(ptr, size, true)) {
-		sdallocx_default(ptr, size, flags);
+	if (likely(flags == 0)) {
+		if (likely(free_fastpath(ptr, size, true))) {
+			return;
+		}
 	}
+	sdallocx_default(ptr, size, flags);
 }
 
 JEMALLOC_ALWAYS_INLINE void JEMALLOC_NOTHROW

include/jemalloc/internal/rtree.h

@@ -166,6 +166,8 @@ bool rtree_new(rtree_t *rtree, base_t *base, bool zeroed);
 
 rtree_leaf_elm_t *rtree_leaf_elm_lookup_hard(tsdn_t *tsdn, rtree_t *rtree,
     rtree_ctx_t *rtree_ctx, uintptr_t key, bool dependent, bool init_missing);
+rtree_contents_t rtree_read_slow(
+    tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, uintptr_t key);
 
 JEMALLOC_ALWAYS_INLINE unsigned
 rtree_calc_leaf_maskbits(void) {
@@ -522,8 +524,8 @@ rtree_read_independent(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx,
 	return false;
 }
 
-static inline rtree_contents_t
-rtree_read(
+JEMALLOC_ALWAYS_INLINE rtree_contents_t
+rtree_read_impl(
     tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, uintptr_t key) {
 	rtree_leaf_elm_t *elm = rtree_leaf_elm_lookup(tsdn, rtree, rtree_ctx,
 	    key, /* dependent */ true, /* init_missing */ false);
@@ -531,6 +533,12 @@ rtree_read(
 	return rtree_leaf_elm_read(tsdn, rtree, elm, /* dependent */ true);
 }
 
+static inline rtree_contents_t
+rtree_read(
+    tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, uintptr_t key) {
+	return rtree_read_impl(tsdn, rtree, rtree_ctx, key);
+}
+
 static inline rtree_metadata_t
 rtree_metadata_read(
     tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, uintptr_t key) {

src/rtree.c

@@ -315,3 +315,9 @@ rtree_ctx_data_init(rtree_ctx_t *ctx) {
 		cache->leaf = NULL;
 	}
 }
+
+rtree_contents_t
+rtree_read_slow(
+    tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, uintptr_t key) {
+	return rtree_read_impl(tsdn, rtree, rtree_ctx, key);
+}