GC: Move zombie page accounting into the default GC

The zombie_objspaces table in gc.c kept a page count for each entry and
a running total. gc.c asked the GC for the counts through
rb_gc_impl_zombie_heap_page_count, and the default GC read the total
back through rb_gc_vm_zombie_total_pages and
rb_gc_vm_refresh_zombie_pages. Only the default GC uses the numbers, so
they crossed the GC API twice for nothing.

In this commit we replace both page count functions with one hook,
rb_gc_impl_objspace_zombify, which gc.c calls when it parks an objspace
in the zombie table. The default GC stores the floored page count in the
objspace and keeps the total in the global objspace. objspace_absorb
subtracts it again, and each global cycle re-measures the zombies in its
snapshot. A nonzero zombie_pages also marks an objspace as a zombie, so
the verifier no longer needs rb_gc_vm_zombie_objspace_p.

The writers keep the same locks as before: the VM lock for zombify and
absorb, the barrier for the refresh. With RGENGC_CHECK_MODE we assert at
each global snapshot that zombie_total_pages equals the sum of
zombie_pages over the enumerated objspaces.

gc.c

@@ -679,7 +679,7 @@ typedef struct gc_function_map {
     bool (*during_postmortem_p)(void *objspace_ptr);
     bool (*obj_foreign_p)(void *objspace_ptr, VALUE obj);
     bool (*shref_marked_p)(void *objspace_ptr, VALUE obj);
-    size_t (*zombie_heap_page_count)(void *objspace_ptr);
+    void (*objspace_zombify)(void *objspace_ptr);
     void (*objspace_absorb)(void *dst_ptr, void *src_ptr);
     void (*gc_rest)(void *objspace_ptr);
     VALUE (*config_get)(void *objpace_ptr);
@@ -875,7 +875,7 @@ ruby_modular_gc_init(void)
     load_modular_gc_func(during_postmortem_p);
     load_modular_gc_func(obj_foreign_p);
     load_modular_gc_func(shref_marked_p);
-    load_modular_gc_func(zombie_heap_page_count);
+    load_modular_gc_func(objspace_zombify);
     load_modular_gc_func(objspace_absorb);
     load_modular_gc_func(gc_rest);
     load_modular_gc_func(config_set);
@@ -980,7 +980,7 @@ ruby_modular_gc_init(void)
 # define rb_gc_impl_during_postmortem_p rb_gc_functions.during_postmortem_p
 # define rb_gc_impl_obj_foreign_p rb_gc_functions.obj_foreign_p
 # define rb_gc_impl_shref_marked_p rb_gc_functions.shref_marked_p
-# define rb_gc_impl_zombie_heap_page_count rb_gc_functions.zombie_heap_page_count
+# define rb_gc_impl_objspace_zombify rb_gc_functions.objspace_zombify
 # define rb_gc_impl_objspace_absorb rb_gc_functions.objspace_absorb
 # define rb_gc_impl_gc_rest rb_gc_functions.gc_rest
 # define rb_gc_impl_config_get rb_gc_functions.config_get
@@ -4249,19 +4249,6 @@ rb_gc_registered_addr_owned_by_registrant_p(VALUE *addr, void *objspace)
     return false;
 }
 
-/* True while the objspace is a zombie pending merge.  Registration ownership moves to
- * the inheritor before the zombie merge completes, so a value still owned by a zombie
- * is a safe transient for the verifier. */
-bool
-rb_gc_vm_zombie_objspace_p(void *objspace)
-{
-    rb_vm_t *vm = GET_VM();
-    for (size_t i = 0; i < vm->gc.zombie_objspaces_count; i++) {
-        if (vm->gc.zombie_objspaces[i].objspace == objspace) return true;
-    }
-    return false;
-}
-
 static bool
 gc_registered_addrs_remove(rb_ractor_t *r, VALUE *addr)
 {
@@ -4460,14 +4447,12 @@ zombie_objspaces_push(rb_vm_t *vm, void *objspace, void **owner_slot, struct rb_
         vm->gc.zombie_objspaces = grown;
         vm->gc.zombie_objspaces_capa = new_capa;
     }
-    size_t pages = rb_gc_impl_zombie_heap_page_count(objspace);
     vm->gc.zombie_objspaces[vm->gc.zombie_objspaces_count++] = (struct rb_objspace_zombie){
         .objspace = objspace,
         .owner_slot = owner_slot,
         .owner = owner,
-        .pages = pages,
     };
-    vm->gc.zombie_total_pages += pages;
+    rb_gc_impl_objspace_zombify(objspace);
 }
 
 /* Called for a Ractor that terminated without being joined.  Its objspace loses its
@@ -4570,7 +4555,6 @@ rb_gc_vm_forget_zombie(void *objspace)
     size_t n = vm->gc.zombie_objspaces_count;
     for (size_t i = 0; i < n; i++) {
         if (vm->gc.zombie_objspaces[i].objspace == objspace) {
-            vm->gc.zombie_total_pages -= vm->gc.zombie_objspaces[i].pages;
             vm->gc.zombie_objspaces[i] = vm->gc.zombie_objspaces[n - 1];
             vm->gc.zombie_objspaces_count = n - 1;
             break;
@@ -4578,15 +4562,6 @@ rb_gc_vm_forget_zombie(void *objspace)
     }
 }
 
-/* Total zombie pages, deciding whether to start a global GC.  An upper bound between
- * global cycles (each re-measures under the barrier), so a stale value cannot
- * re-trigger; a lock-free read at worst fires one cycle early or late. */
-size_t
-rb_gc_vm_zombie_total_pages(void)
-{
-    return GET_VM()->gc.zombie_total_pages;
-}
-
 /* Number of live Ractors, for the heap growth heuristic (r_mul); a racy read is fine. */
 unsigned int
 rb_gc_vm_ractor_count(void)
@@ -4594,20 +4569,6 @@ rb_gc_vm_ractor_count(void)
     return GET_VM()->ractor.cnt;
 }
 
-/* Called by a global cycle from inside the barrier. */
-void
-rb_gc_vm_refresh_zombie_pages(void)
-{
-    rb_vm_t *vm = GET_VM();
-    size_t total = 0;
-    for (size_t i = 0; i < vm->gc.zombie_objspaces_count; i++) {
-        size_t pages = rb_gc_impl_zombie_heap_page_count(vm->gc.zombie_objspaces[i].objspace);
-        vm->gc.zombie_objspaces[i].pages = pages;
-        total += pages;
-    }
-    vm->gc.zombie_total_pages = total;
-}
-
 void
 rb_gc_rest(void)
 {
@@ -4725,9 +4686,8 @@ objspace_absorb_disowned_zombies(void)
         while (i < vm->gc.zombie_objspaces_count) {
             if (vm->gc.zombie_objspaces[i].owner_slot == NULL) {
                 void *zombie = vm->gc.zombie_objspaces[i].objspace;
-                /* Remove via forget, which also subtracts the entry's pages from
-                 * zombie_total_pages; a hand-written swap-remove would leave a phantom
-                 * total that keeps starting stop-the-world global cycles. */
+                /* Drop the entry before the merge, in this same VM-lock section:
+                 * objspace_absorb expects src to be out of the table. */
                 gc_absorbing_zombie++;
                 rb_gc_vm_forget_zombie(zombie);
                 objspace_absorb_merge(rb_gc_get_objspace(), zombie);

gc/default/default.c

@@ -669,6 +669,9 @@ typedef struct rb_objspace {
     rb_heap_t heaps[HEAP_COUNT];
     size_t empty_pages_count;
     struct heap_page *empty_pages;
+    /* Nonzero while gc.c parks this objspace in its zombie table: the pages it adds to
+     * global_objspace->zombie_total_pages.  Written under the VM lock or the barrier. */
+    size_t zombie_pages;
 
     struct {
         rb_atomic_t finalizing;
@@ -866,8 +869,10 @@ typedef struct rb_global_objspace {
         size_t arenas_unmapped;           /* cumulative arenas munmapped (for GC.stat) */
     } page_pool;
 
-    /* Zombie pages left after the last global cycle (roughly the live data).  Updated
-     * under the barrier; readers (gc_need_global_p) may be racy. */
+    /* Pages held by zombie objspaces (sum of their zombie_pages), and what was left of
+     * it after the last global cycle (roughly the live data).  Written under the VM
+     * lock or the barrier; readers (gc_need_global_p) may be racy. */
+    size_t zombie_total_pages;
     size_t zombie_pages_survivors;
 
     /* An objspace merge (objspace_absorb) is running: suppress the cross-objspace
@@ -7339,7 +7344,7 @@ verify_registered_addr(VALUE *slot, VALUE initial_value, void *owner_objspace, v
     /* Join and orphan handling move a registration to the inheritor before the
      * source objspace merge; a global GC scans every registry while the zombie
      * exists, so this is a safe transient exemption. */
-    if (rb_gc_vm_zombie_objspace_p(value_objspace)) return;
+    if (value_objspace->zombie_pages != 0) return;
     if (value_objspace->flags.during_postmortem) return;
     if (MARKED_IN_BITMAP(GET_HEAP_SHAREABLE_BITS(v), v)) return;
     if (MARKED_IN_BITMAP(GET_HEAP_SHREF_BITS(v), v)) return;
@@ -8860,7 +8865,7 @@ gc_need_global_p(rb_objspace_t *objspace)
      * is live data, so retrigger only once TRIGGER more pages accumulate on top of it.
      * Otherwise one live-heavy unjoined zombie turns every GC stop-the-world forever. */
     {
-        size_t zp = rb_gc_vm_zombie_total_pages();
+        size_t zp = global_objspace->zombie_total_pages;
         size_t base = global_objspace->zombie_pages_survivors < zp ? global_objspace->zombie_pages_survivors : zp;
         if (zp - base >= ZOMBIE_PAGES_TRIGGER) return true;
     }
@@ -9535,18 +9540,44 @@ rb_gc_impl_shref_marked_p(void *objspace_ptr, VALUE obj)
     return MARKED_IN_BITMAP(GET_HEAP_SHREF_BITS(obj), obj) != 0;
 }
 
-/* What a zombie objspace is worth to the zombie_objspaces page accounting, floored:
- * the retire GC can empty an ownerless objspace outright, yet its struct, mark stack
- * and heap_page structs stay in malloc and it keeps the world multi-objspace until a
- * global GC merges it, so a page-less zombie must still push the zombie trigger. */
-size_t
-rb_gc_impl_zombie_heap_page_count(void *objspace_ptr)
+/* What a zombie objspace is worth to the zombie page trigger, floored: the retire GC
+ * can empty an ownerless objspace outright, yet its struct, mark stack and heap_page
+ * structs stay in malloc and it keeps the world multi-objspace until a global GC
+ * merges it, so a page-less zombie must still push the trigger.  Never 0, so a
+ * nonzero zombie_pages also marks the objspace as a zombie. */
+static size_t
+objspace_zombie_pages(rb_objspace_t *objspace)
 {
-    rb_objspace_t *objspace = objspace_ptr;
     size_t pages = rb_darray_size(objspace->heap_pages.sorted);
     return pages > HEAP_COUNT ? pages : HEAP_COUNT;
 }
 
+/* gc.c parked the objspace in its zombie table, under the VM lock. */
+void
+rb_gc_impl_objspace_zombify(void *objspace_ptr)
+{
+    rb_objspace_t *objspace = objspace_ptr;
+    GC_ASSERT(objspace->zombie_pages == 0);
+    objspace->zombie_pages = objspace_zombie_pages(objspace);
+    global_objspace->zombie_total_pages += objspace->zombie_pages;
+}
+
+/* Re-measure every zombie this cycle covered, by difference, so a zombie parked after
+ * the snapshot keeps the pages it was parked with.  Inside the barrier. */
+static void
+gc_global_refresh_zombie_pages(void)
+{
+    for (size_t i = 0; i < global_objspace->global_gc.n_objspaces; i++) {
+        rb_objspace_t *objspace = global_objspace->global_gc.objspaces[i];
+        if (objspace->zombie_pages == 0) continue;
+        size_t pages = objspace_zombie_pages(objspace);
+        global_objspace->zombie_total_pages -= objspace->zombie_pages;
+        global_objspace->zombie_total_pages += pages;
+        objspace->zombie_pages = pages;
+    }
+    global_objspace->zombie_pages_survivors = global_objspace->zombie_total_pages;
+}
+
 static void
 gc_global_objspaces_i(void *os, void *data)
 {
@@ -9571,11 +9602,14 @@ gc_global_snapshot_objspaces(void)
 #if RGENGC_CHECK_MODE
     /* Check that the incrementally maintained page_index agrees with the per-objspace
      * sorted arrays. */
-    size_t total = 0;
+    size_t total = 0, zombie_total = 0;
     for (size_t i = 0; i < global_objspace->global_gc.n_objspaces; i++) {
         total += rb_darray_size(global_objspace->global_gc.objspaces[i]->heap_pages.sorted);
+        zombie_total += global_objspace->global_gc.objspaces[i]->zombie_pages;
     }
     GC_ASSERT(total == global_objspace->page_index.n_pages);
+    /* Every zombie is enumerated, so the snapshot sees all of zombie_total_pages. */
+    GC_ASSERT(zombie_total == global_objspace->zombie_total_pages);
 #endif
 }
 
@@ -9949,11 +9983,10 @@ gc_start_global(rb_objspace_t *driver, unsigned int reason, bool compact, bool a
      * next absorb). */
     rb_gc_reset_absorbed_since_global_gc();
 
-    /* Re-measure the zombie_objspaces table now that the garbage is gone; entries are stable
-     * inside the barrier.  Without this, the page trigger above keeps firing on the stale
-     * numbers left when a joinable (slotted) zombie retires without any pass merging it. */
-    rb_gc_vm_refresh_zombie_pages();
-    global_objspace->zombie_pages_survivors = rb_gc_vm_zombie_total_pages();
+    /* Re-measure the zombies now that the garbage is gone.  Without this, the page
+     * trigger above keeps firing on the stale numbers left when a joinable (slotted)
+     * zombie retires without any pass merging it. */
+    gc_global_refresh_zombie_pages();
 
     /* If the sweep above collected an unjoined Ractor object, ractor_free disowned its
      * zombie_objspaces entry and posted the merge to main as a postponed job; the objspace
@@ -10019,6 +10052,13 @@ objspace_absorb(rb_objspace_t *dst, rb_objspace_t *src)
     const bool prev_absorb = global_objspace->during_absorb;
     global_objspace->during_absorb = true;
 
+    /* gc.c dropped src from its zombie table in this same VM-lock section, so src went
+     * through zombify and its pages are part of the total. */
+    GC_ASSERT(src->zombie_pages != 0);
+    GC_ASSERT(global_objspace->zombie_total_pages >= src->zombie_pages);
+    global_objspace->zombie_total_pages -= src->zombie_pages;
+    src->zombie_pages = 0;
+
     /* Settle dst first: adding pages under a walking lazy-sweep cursor, or into a
      * half-marked incremental heap, would sweep the merged pages with src's stale mark
      * bits and free live objects. */

gc/gc.h

@@ -79,10 +79,7 @@ MODULAR_GC_FN void rb_gc_vm_each_objspace(void (*func)(void *objspace, void *dat
 MODULAR_GC_FN void rb_gc_each_registered_addr(rb_gc_registered_addr_cb func, void *data);
 /* Verifier support, valid only with the world stopped (no extra locking inside). */
 MODULAR_GC_FN bool rb_gc_registered_addr_owned_by_registrant_p(VALUE *addr, void *objspace);
-MODULAR_GC_FN bool rb_gc_vm_zombie_objspace_p(void *objspace);
-MODULAR_GC_FN size_t rb_gc_vm_zombie_total_pages(void);
 MODULAR_GC_FN unsigned int rb_gc_vm_ractor_count(void);
-MODULAR_GC_FN void rb_gc_vm_refresh_zombie_pages(void);
 /* No MODULAR_GC_FN: the VM side (ractor.c) calls this too, so it needs external
  * linkage even in a non-modular build (see internal/gc.h). */
 bool rb_gc_single_objspace_p(void);

gc/gc_impl.h

@@ -153,7 +153,11 @@ GC_IMPL_FN bool rb_gc_impl_during_postmortem_p(void *objspace_ptr);
  * objspace impl. */
 GC_IMPL_FN bool rb_gc_impl_obj_foreign_p(void *objspace_ptr, VALUE obj);
 GC_IMPL_FN bool rb_gc_impl_shref_marked_p(void *objspace_ptr, VALUE obj);
-GC_IMPL_FN size_t rb_gc_impl_zombie_heap_page_count(void *objspace_ptr);
+/* gc.c parked objspace_ptr in its zombie table: a terminated Ractor's objspace that
+ * rb_gc_impl_objspace_absorb later merges.  Called under the VM lock, possibly from
+ * inside a global GC sweep (an unjoined Ractor's dfree disowns its objspace), so it
+ * must not allocate through the GC or expect to take a barrier. */
+GC_IMPL_FN void rb_gc_impl_objspace_zombify(void *objspace_ptr);
 GC_IMPL_FN void rb_gc_impl_objspace_absorb(void *dst_ptr, void *src_ptr);
 GC_IMPL_FN void rb_gc_impl_each_object(void *objspace_ptr, void (*func)(VALUE obj, void *data), void *data);
 // Finalizers

gc/mmtk/mmtk.c

@@ -1858,11 +1858,10 @@ rb_gc_impl_shref_marked_p(void *objspace_ptr, VALUE obj)
     return true;
 }
 
-size_t
-rb_gc_impl_zombie_heap_page_count(void *objspace_ptr)
+void
+rb_gc_impl_objspace_zombify(void *objspace_ptr)
 {
     /* With a single objspace zombie_objspaces is always empty. */
-    return 0;
 }
 
 void

gc/wbcheck/wbcheck.c

@@ -1557,10 +1557,9 @@ rb_gc_impl_shref_marked_p(void *objspace_ptr, VALUE obj)
     return true;
 }
 
-size_t
-rb_gc_impl_zombie_heap_page_count(void *objspace_ptr)
+void
+rb_gc_impl_objspace_zombify(void *objspace_ptr)
 {
-    return 0;
 }
 
 void

vm_core.h

@@ -809,16 +809,9 @@ typedef struct rb_vm_struct {
              * rb_gc_register_mark_object pins and join value through it.  NULL for an
              * orphan, whose Ractor struct is gone and has neither any more. */
             struct rb_ractor_struct *owner;
-            /* Heap pages this zombie holds: measured when it retires and refreshed
-             * under the barrier of each global cycle.  The total below stays exactly
-             * in sync, entry by entry. */
-            size_t pages;
         } *zombie_objspaces;
         size_t zombie_objspaces_count;
         size_t zombie_objspaces_capa;
-        /* Sum of .pages over zombie_objspaces.  Between global cycles it is an upper
-         * bound: a zombie's heap never grows and only shrinks at a global cycle. */
-        size_t zombie_total_pages;
 
 #if USE_MODULAR_GC
         struct gc_mark_func_data_struct *mark_func_data;