diff --git a/gc.c b/gc.c index aee89616a9a..c465ce0ba29 100644 --- a/gc.c +++ b/gc.c @@ -690,7 +690,7 @@ typedef struct gc_function_map { // Object allocation VALUE (*new_obj)(void *objspace_ptr, void *cache_ptr, VALUE klass, VALUE flags, bool wb_protected, size_t alloc_size, size_t *actual_alloc_size); bool (*zjit_new_obj_fastpath)(void *objspace_ptr, size_t alloc_size, VALUE flags, VALUE klass, struct rb_gc_zjit_fastpath *fastpath); - const uintptr_t *(*zjit_writebarrier_required_p_ptr)(void *objspace_ptr); + int32_t (*zjit_writebarrier_required_offset)(void); size_t (*obj_slot_size)(VALUE obj); size_t (*size_slot_size)(void *objspace_ptr, size_t size); bool (*size_allocatable_p)(size_t size); @@ -886,7 +886,7 @@ ruby_modular_gc_init(void) // Object allocation load_modular_gc_func(new_obj); load_modular_gc_func(zjit_new_obj_fastpath); - load_modular_gc_func(zjit_writebarrier_required_p_ptr); + load_modular_gc_func(zjit_writebarrier_required_offset); load_modular_gc_func(obj_slot_size); load_modular_gc_func(size_slot_size); load_modular_gc_func(size_allocatable_p); @@ -991,7 +991,7 @@ ruby_modular_gc_init(void) // Object allocation # define rb_gc_impl_new_obj rb_gc_functions.new_obj # define rb_gc_impl_zjit_new_obj_fastpath rb_gc_functions.zjit_new_obj_fastpath -# define rb_gc_impl_zjit_writebarrier_required_p_ptr rb_gc_functions.zjit_writebarrier_required_p_ptr +# define rb_gc_impl_zjit_writebarrier_required_offset rb_gc_functions.zjit_writebarrier_required_offset # define rb_gc_impl_obj_slot_size rb_gc_functions.obj_slot_size # define rb_gc_impl_size_slot_size rb_gc_functions.size_slot_size # define rb_gc_impl_size_allocatable_p rb_gc_functions.size_allocatable_p @@ -4152,10 +4152,10 @@ rb_gc_zjit_new_obj_fastpath(size_t alloc_size, VALUE flags, VALUE klass, struct #endif } -const uintptr_t * -rb_gc_zjit_writebarrier_required_p_ptr(void) +int32_t +rb_gc_zjit_writebarrier_required_offset(void) { - return rb_gc_impl_zjit_writebarrier_required_p_ptr(rb_gc_get_objspace()); + return rb_gc_impl_zjit_writebarrier_required_offset(); } /* diff --git a/gc/default/default.c b/gc/default/default.c index 74cc688da19..98ed3ba8de3 100644 --- a/gc/default/default.c +++ b/gc/default/default.c @@ -660,7 +660,6 @@ typedef struct rb_objspace { unsigned int gc_lock_barrier : 1; unsigned int during_reference_updating : 1; unsigned int during_minor_gc : 1; - unsigned int during_incremental_marking : 1; unsigned int during_postmortem : 1; unsigned int measure_gc : 1; } flags; @@ -832,6 +831,12 @@ typedef struct rb_objspace { rb_nativethread_lock_t lock; struct gc_process_stat_snapshot published; } process_stat; + + /* The incremental-marking flag as an addressable field, so that ZJIT-generated + * code can read it; see rb_gc_impl_zjit_writebarrier_required_offset. + * Single-writer: a local GC runs on the owning Ractor's thread and a global GC + * writes under the stop-the-world barrier. Read by JIT code on the same thread. */ + bool zjit_during_incremental_marking; } rb_objspace_t; /* The one VM-global GC structure; for now it only holds the page pool. Page bodies are @@ -1538,23 +1543,14 @@ total_final_slots_count(rb_objspace_t *objspace) #define is_marking(objspace) (gc_mode(objspace) == gc_mode_marking) #define is_sweeping(objspace) (gc_mode(objspace) == gc_mode_sweeping) #define is_full_marking(objspace) ((objspace)->flags.during_minor_gc == FALSE) -#define is_incremental_marking(objspace) ((objspace)->flags.during_incremental_marking != FALSE) +#define is_incremental_marking(objspace) ((objspace)->zjit_during_incremental_marking != FALSE) #define will_be_incremental_marking(objspace) ((objspace)->rgengc.need_major_gc != GPR_FLAG_NONE) -/* Mirrors flags.during_incremental_marking so that ZJIT-generated code can - * read it (a bitfield has no address); see rb_gc_impl_zjit_writebarrier_required_p_ptr. - * Only transitions write to it: incremental marking requires a single objspace, - * so a redundant FALSE store from another objspace's GC entry must not clobber - * the marking objspace's TRUE. Written inside GC while mutators are stopped; - * read by JIT code. */ -static uintptr_t zjit_during_incremental_marking = FALSE; - static void gc_set_incremental_marking(rb_objspace_t *objspace, unsigned int value) { - if (objspace->flags.during_incremental_marking != value) { - objspace->flags.during_incremental_marking = value; - zjit_during_incremental_marking = value; + if (objspace->zjit_during_incremental_marking != value) { + objspace->zjit_during_incremental_marking = value; } } /* @@ -3476,10 +3472,10 @@ rb_gc_impl_zjit_new_obj_fastpath(void *objspace_ptr, size_t alloc_size, VALUE fl #endif } -const uintptr_t * -rb_gc_impl_zjit_writebarrier_required_p_ptr(void *objspace_ptr) +int32_t +rb_gc_impl_zjit_writebarrier_required_offset(void) { - return &zjit_during_incremental_marking; + return (int32_t)offsetof(rb_objspace_t, zjit_during_incremental_marking); } NOINLINE(static VALUE newobj_refill(rb_objspace_t *objspace, size_t heap_idx)); diff --git a/gc/gc_impl.h b/gc/gc_impl.h index 80da58d2692..367b83c327a 100644 --- a/gc/gc_impl.h +++ b/gc/gc_impl.h @@ -89,14 +89,15 @@ GC_IMPL_FN VALUE rb_gc_impl_new_obj(void *objspace_ptr, void *cache_ptr, VALUE k * generated instruction sequence; see zjit/src/gc_fastpath.rs. */ GC_IMPL_FN bool rb_gc_impl_zjit_new_obj_fastpath(void *objspace_ptr, size_t alloc_size, VALUE flags, VALUE klass, struct rb_gc_zjit_fastpath *fastpath); -/* Returns a pointer to a word that is nonzero while incremental marking is in - * progress, or NULL if the implementation does not support ZJIT's inline write - * barrier fast path. ZJIT-generated code skips the call to rb_gc_writebarrier() - * when the receiver has neither RUBY_FL_PROMOTED nor RUBY_FL_SHAREABLE set and - * this word is zero, so only implementations whose write barrier is a no-op - * under exactly those conditions may return non-NULL. +/* Returns the offset of a word inside the objspace that is nonzero while that + * objspace is incremental-marking, or -1 if the implementation does not support + * ZJIT's inline write barrier fast path. ZJIT-generated code skips the call to + * rb_gc_writebarrier() when the receiver has neither RUBY_FL_PROMOTED nor + * RUBY_FL_SHAREABLE set and the word of the current Ractor's objspace is zero, + * so only implementations whose write barrier is a no-op under exactly those + * conditions may return a nonnegative offset. */ -GC_IMPL_FN const uintptr_t *rb_gc_impl_zjit_writebarrier_required_p_ptr(void *objspace_ptr); +GC_IMPL_FN int32_t rb_gc_impl_zjit_writebarrier_required_offset(void); GC_IMPL_FN size_t rb_gc_impl_obj_slot_size(VALUE obj); GC_IMPL_FN size_t rb_gc_impl_size_slot_size(void *objspace_ptr, size_t size); GC_IMPL_FN bool rb_gc_impl_size_allocatable_p(size_t size); diff --git a/gc/mmtk/mmtk.c b/gc/mmtk/mmtk.c index cf80d985de2..194b2d3c3c9 100644 --- a/gc/mmtk/mmtk.c +++ b/gc/mmtk/mmtk.c @@ -742,12 +742,12 @@ rb_gc_impl_zjit_new_obj_fastpath(void *objspace_ptr, size_t alloc_size, VALUE fl #endif } -const uintptr_t * -rb_gc_impl_zjit_writebarrier_required_p_ptr(void *objspace_ptr) +int32_t +rb_gc_impl_zjit_writebarrier_required_offset(void) { /* MMTk's write barrier is not a no-op for young, unshareable receivers, * so ZJIT must always call rb_gc_writebarrier(). */ - return NULL; + return -1; } void diff --git a/gc/wbcheck/wbcheck.c b/gc/wbcheck/wbcheck.c index 22555c436f1..a5fd562f593 100644 --- a/gc/wbcheck/wbcheck.c +++ b/gc/wbcheck/wbcheck.c @@ -1573,11 +1573,11 @@ rb_gc_impl_obj_became_shareable(void *objspace_ptr, VALUE obj) { } -const uintptr_t * -rb_gc_impl_zjit_writebarrier_required_p_ptr(void *objspace_ptr) +int32_t +rb_gc_impl_zjit_writebarrier_required_offset(void) { /* ZJIT must always call rb_gc_writebarrier() so wbcheck sees every barrier. */ - return NULL; + return -1; } void diff --git a/internal/gc.h b/internal/gc.h index 2bc2070cf0a..e167df51b23 100644 --- a/internal/gc.h +++ b/internal/gc.h @@ -211,7 +211,7 @@ VALUE rb_newobj_of_unprotected(VALUE klass, VALUE flags, size_t size); void *rb_gc_ractor_cache_alloc(struct rb_ractor_struct *ractor); void rb_gc_ractor_cache_free(void *cache); bool rb_gc_zjit_new_obj_fastpath(size_t alloc_size, VALUE flags, VALUE klass, struct rb_gc_zjit_fastpath *fastpath); -const uintptr_t *rb_gc_zjit_writebarrier_required_p_ptr(void); +int32_t rb_gc_zjit_writebarrier_required_offset(void); bool rb_gc_size_allocatable_p(size_t size); size_t rb_gc_size_slot_size(size_t size); diff --git a/zjit/src/codegen.rs b/zjit/src/codegen.rs index 88b8e535c20..ae1c6b09062 100644 --- a/zjit/src/codegen.rs +++ b/zjit/src/codegen.rs @@ -1512,17 +1512,23 @@ fn gen_write_barrier(jit: &mut JITState, asm: &mut Assembler, recv: Opnd, val: O asm.je(jit, result_edge.clone()); unsafe extern "C" { - fn rb_gc_zjit_writebarrier_required_p_ptr() -> *const usize; + fn rb_gc_zjit_writebarrier_required_offset() -> i32; } // The barrier is a no-op when the receiver is young (not RUBY_FL_PROMOTED), - // not RUBY_FL_SHAREABLE, and incremental marking is off; see - // rb_gc_impl_writebarrier. NULL means the GC has no such fast path. - let marking_ptr = unsafe { rb_gc_zjit_writebarrier_required_p_ptr() }; - if !marking_ptr.is_null() { + // not RUBY_FL_SHAREABLE, and the current Ractor's objspace is not + // incremental-marking; see rb_gc_impl_writebarrier. A negative offset + // means the GC has no such fast path. The word is per-objspace: compiled + // code is shared across Ractors, so find the objspace through EC. + let wb_offset = unsafe { rb_gc_zjit_writebarrier_required_offset() }; + if wb_offset >= 0 { let flags = asm.load(Opnd::mem(64, recv, RUBY_OFFSET_RBASIC_FLAGS)); let old_or_shareable = asm.and(flags, Opnd::UImm((RUBY_FL_PROMOTED | RUBY_FL_SHAREABLE) as u64)); - let marking_addr = asm.load(Opnd::const_ptr(marking_ptr)); - let marking = asm.load(Opnd::mem(64, marking_addr, 0)); + let thread = asm.load(Opnd::mem(64, EC, RUBY_OFFSET_EC_THREAD_PTR as i32)); + let ractor = asm.load(Opnd::mem(64, thread, RUBY_OFFSET_THREAD_RACTOR as i32)); + let objspace = asm.load(Opnd::mem(64, ractor, unsafe { rb_zjit_runtime_offsets.ractor_objspace })); + let marking = asm.load(Opnd::mem(8, objspace, wb_offset)); + let marking = marking.with_num_bits(64); + let marking = asm.and(marking, 0xFF.into()); let needs_wb = asm.or(old_or_shareable, marking); asm.test(needs_wb, needs_wb); asm.jz(jit, result_edge.clone());