mirrors/ruby
0
0 Fork 0
mirror of https://github.com/ruby/ruby.git synced 2026-10-29 00:46:26 +08:00
Code Activity
Files
master
ruby/gc/default/default.c
T
Luke Gruber 29264f878a Fix rb_gc_ever_multi_ractor_p to simply read the latch variable 
In single-ractor mode, we don't want foreign_object_p checks to actually
run (there can be no foreign objspaces). This foreign_object_p check return
false if there was never a (non-main) Ractor created. Previously,
rb_gc_ever_multi_ractor_p() was needlessly complex. This change should
speed up marking and WB implementations in single-ractor mode.
2026-10-26 14:29:57 -04:00

13651 lines
464 KiB
C
Raw Permalink Blame History

#include "ruby/internal/config.h"
#include <signal.h>
#include <string.h>
#ifndef _WIN32
# include <sys/mman.h>
# include <unistd.h>
# include <fcntl.h>
# ifdef HAVE_SYS_PRCTL_H
# include <sys/prctl.h>
# endif
#endif
#if !defined(PAGE_SIZE) && defined(HAVE_SYS_USER_H)
/* LIST_HEAD conflicts with sys/queue.h on macOS */
# include <sys/user.h>
#endif
#ifdef BUILDING_MODULAR_GC
# define nlz_int64(x) (x == 0 ? 64 : (unsigned int)__builtin_clzll((unsigned long long)x))
# define rb_popcount_intptr(x) ((unsigned int)__builtin_popcountll((unsigned long long)(x)))
#else
# include "internal/bits.h"
#endif
#include "ruby/ruby.h"
#include "ruby/atomic.h"
#include "ruby_atomic.h"
#include "ruby/debug.h"
#include "ruby/thread.h"
#include "ruby/util.h"
#include "ruby/vm.h"
#include "ruby/internal/encoding/string.h"
#include "ccan/list/list.h"
#include "darray.h"
#include "gc/gc.h"
#include "gc/gc_impl.h"
#include "yjit.h"
#include "zjit.h"
#include "internal/static_assert.h"
#include "internal/vm_map.h"
#ifdef BUILDING_MODULAR_GC
/* hrtime.h transitively includes internal/time.h -> internal/bits.h, which are
* not available to out-of-tree modular GC builds. We only use a monotonic
* clock plus saturating add/sub, so provide that subset locally with the same
* semantics as hrtime.h. */
# include <time.h>
typedef uint64_t rb_hrtime_t;
# define RB_HRTIME_PER_SEC ((rb_hrtime_t)1000000000)
static inline rb_hrtime_t
rb_hrtime_now(void)
{
# if defined(HAVE_CLOCK_GETTIME) && defined(CLOCK_MONOTONIC)
struct timespec ts;
if (clock_gettime(CLOCK_MONOTONIC, &ts) == 0) {
return (rb_hrtime_t)ts.tv_sec * RB_HRTIME_PER_SEC + (rb_hrtime_t)ts.tv_nsec;
}
# endif
return 0;
}
static inline rb_hrtime_t
rb_hrtime_add(rb_hrtime_t a, rb_hrtime_t b)
{
rb_hrtime_t c = a + b;
return c < a ? UINT64_MAX : c; /* saturate on overflow */
}
static inline rb_hrtime_t
rb_hrtime_sub(rb_hrtime_t a, rb_hrtime_t b)
{
return a < b ? 0 : a - b;
}
#else
# include "hrtime.h"
#endif
#include "probes.h"
/* cl.exe's traditional preprocessor passes __VA_ARGS__ to a nested macro as
* a single argument; the extra expansion re-scans it into separate ones. */
#define RUBY_DTRACE_GC_HOOK_EXPAND(expr) expr
#define RUBY_DTRACE_GC_HOOK(name, ...) \
do {if (RUBY_DTRACE_GC_##name##_ENABLED()) RUBY_DTRACE_GC_HOOK_EXPAND(RUBY_DTRACE_GC_##name(__VA_ARGS__));} while (0)
#if USE_ZJIT
# include "gc/default/zjit_fastpath.h"
#endif
#ifdef BUILDING_MODULAR_GC
# define RB_DEBUG_COUNTER_INC(_name) ((void)0)
# define RB_DEBUG_COUNTER_INC_IF(_name, cond) (!!(cond))
#else
# include "debug_counter.h"
#endif
#ifdef BUILDING_MODULAR_GC
# define rb_asan_poison_object(obj) ((void)(obj))
# define rb_asan_unpoison_object(obj, newobj_p) ((void)(obj), (void)(newobj_p))
# define asan_unpoisoning_object(obj) if ((obj) || true)
# define asan_poison_memory_region(ptr, size) ((void)(ptr), (void)(size))
# define asan_unpoison_memory_region(ptr, size, malloc_p) ((void)(ptr), (size), (malloc_p))
# define asan_unpoisoning_memory_region(ptr, size) if ((ptr) || (size) || true)
# define VALGRIND_MAKE_MEM_DEFINED(ptr, size) ((void)(ptr), (void)(size))
# define VALGRIND_MAKE_MEM_UNDEFINED(ptr, size) ((void)(ptr), (void)(size))
#else
# include "internal/sanitizers.h"
#endif
/* MALLOC_HEADERS_BEGIN */
#ifndef HAVE_MALLOC_USABLE_SIZE
# ifdef _WIN32
# define HAVE_MALLOC_USABLE_SIZE
# define malloc_usable_size(a) _msize(a)
# elif defined HAVE_MALLOC_SIZE
# define HAVE_MALLOC_USABLE_SIZE
# define malloc_usable_size(a) malloc_size(a)
# endif
#endif
#ifdef HAVE_MALLOC_USABLE_SIZE
# ifdef RUBY_ALTERNATIVE_MALLOC_HEADER
/* Alternative malloc header is included in ruby/missing.h */
# elif defined(HAVE_MALLOC_H)
# include <malloc.h>
# elif defined(HAVE_MALLOC_NP_H)
# include <malloc_np.h>
# elif defined(HAVE_MALLOC_MALLOC_H)
# include <malloc/malloc.h>
# endif
#endif
#ifdef HAVE_MALLOC_TRIM
# include <malloc.h>
# ifdef __EMSCRIPTEN__
/* malloc_trim is defined in emscripten/emmalloc.h on emscripten. */
# include <emscripten/emmalloc.h>
# endif
#endif
#ifdef HAVE_MACH_TASK_EXCEPTION_PORTS
# include <mach/task.h>
# include <mach/mach_init.h>
# include <mach/mach_port.h>
#endif
#ifndef RUBY_DEBUG_LOG
# define RUBY_DEBUG_LOG(...)
#endif
#ifndef GC_HEAP_INIT_BYTES
#define GC_HEAP_INIT_BYTES (2560 * 1024)
#endif
#ifndef GC_HEAP_FREE_SLOTS
#define GC_HEAP_FREE_SLOTS 4096
#endif
#ifndef GC_RACTOR_HEAP_INIT_BYTES
/* 0 is resolved at boot to the smallest size that works. */
#define GC_RACTOR_HEAP_INIT_BYTES 0
#endif
#ifndef GC_HEAP_GROWTH_FACTOR
#define GC_HEAP_GROWTH_FACTOR 1.8
#endif
#ifndef GC_HEAP_GROWTH_MAX_BYTES
#define GC_HEAP_GROWTH_MAX_BYTES 0 /* 0 is disable */
#endif
#ifndef GC_HEAP_REMEMBERED_WB_UNPROTECTED_OBJECTS_LIMIT_RATIO
# define GC_HEAP_REMEMBERED_WB_UNPROTECTED_OBJECTS_LIMIT_RATIO 0.01
#endif
#ifndef GC_HEAP_OLDOBJECT_LIMIT_FACTOR
#define GC_HEAP_OLDOBJECT_LIMIT_FACTOR 2.0
#endif
#ifndef GC_HEAP_FREE_SLOTS_MIN_RATIO
#define GC_HEAP_FREE_SLOTS_MIN_RATIO 0.20
#endif
#ifndef GC_HEAP_FREE_SLOTS_GOAL_RATIO
#define GC_HEAP_FREE_SLOTS_GOAL_RATIO 0.40
#endif
#ifndef GC_HEAP_FREE_SLOTS_MAX_RATIO
#define GC_HEAP_FREE_SLOTS_MAX_RATIO 0.65
#endif
#ifndef GC_MALLOC_LIMIT_MIN
#define GC_MALLOC_LIMIT_MIN (16 * 1024 * 1024 /* 16MB */)
#endif
#ifndef GC_MALLOC_LIMIT_MAX
#define GC_MALLOC_LIMIT_MAX (32 * 1024 * 1024 /* 32MB */)
#endif
#ifndef GC_MALLOC_LIMIT_GROWTH_FACTOR
#define GC_MALLOC_LIMIT_GROWTH_FACTOR 1.4
#endif
#ifndef GC_OLDMALLOC_LIMIT_MIN
#define GC_OLDMALLOC_LIMIT_MIN (16 * 1024 * 1024 /* 16MB */)
#endif
#ifndef GC_OLDMALLOC_LIMIT_GROWTH_FACTOR
#define GC_OLDMALLOC_LIMIT_GROWTH_FACTOR 1.2
#endif
#ifndef GC_OLDMALLOC_LIMIT_MAX
#define GC_OLDMALLOC_LIMIT_MAX (128 * 1024 * 1024 /* 128MB */)
#endif
#ifndef GC_MALLOC_INCREASE_LOCAL_THRESHOLD
#define GC_MALLOC_INCREASE_LOCAL_THRESHOLD (8 * 1024 /* 8KB */)
#endif
#ifdef RB_THREAD_LOCAL_SPECIFIER
#define USE_MALLOC_INCREASE_LOCAL 1
static RB_THREAD_LOCAL_SPECIFIER int malloc_increase_local;
#else
#define USE_MALLOC_INCREASE_LOCAL 0
#endif
#ifndef GC_CAN_COMPILE_COMPACTION
#if defined(__wasi__) /* WebAssembly doesn't support signals */
# define GC_CAN_COMPILE_COMPACTION 0
#else
# define GC_CAN_COMPILE_COMPACTION 1
#endif
#endif
#ifndef PRINT_ENTER_EXIT_TICK
# define PRINT_ENTER_EXIT_TICK 0
#endif
#ifndef PRINT_ROOT_TICKS
#define PRINT_ROOT_TICKS 0
#endif
#define USE_TICK_T (PRINT_ENTER_EXIT_TICK || PRINT_ROOT_TICKS)
#ifndef HEAP_COUNT
# if SIZEOF_VALUE >= 8
# define HEAP_COUNT 12
# else
# define HEAP_COUNT 5
# endif
#endif
/* The reciprocal table and pool_slot_sizes array are both generated from this
* single definition, so they can never get out of sync. */
#if SIZEOF_VALUE >= 8
# define EACH_POOL_SLOT_SIZE(SLOT) \
SLOT(32) SLOT(40) SLOT(64) SLOT(80) SLOT(96) SLOT(128) \
SLOT(160) SLOT(256) SLOT(512) SLOT(640) SLOT(768) SLOT(1024)
#else
# define EACH_POOL_SLOT_SIZE(SLOT) \
SLOT(32) SLOT(64) SLOT(128) SLOT(256) SLOT(512)
#endif
typedef struct {
size_t heap_init_bytes;
size_t ractor_heap_init_bytes;
size_t heap_free_slots;
double growth_factor;
size_t growth_max_bytes;
double heap_free_slots_min_ratio;
double heap_free_slots_goal_ratio;
double heap_free_slots_max_ratio;
double uncollectible_wb_unprotected_objects_limit_ratio;
double oldobject_limit_factor;
size_t malloc_limit_min;
size_t malloc_limit_max;
double malloc_limit_growth_factor;
size_t oldmalloc_limit_min;
size_t oldmalloc_limit_max;
double oldmalloc_limit_growth_factor;
} ruby_gc_params_t;
static ruby_gc_params_t gc_params = {
GC_HEAP_INIT_BYTES,
GC_RACTOR_HEAP_INIT_BYTES,
GC_HEAP_FREE_SLOTS,
GC_HEAP_GROWTH_FACTOR,
GC_HEAP_GROWTH_MAX_BYTES,
GC_HEAP_FREE_SLOTS_MIN_RATIO,
GC_HEAP_FREE_SLOTS_GOAL_RATIO,
GC_HEAP_FREE_SLOTS_MAX_RATIO,
GC_HEAP_REMEMBERED_WB_UNPROTECTED_OBJECTS_LIMIT_RATIO,
GC_HEAP_OLDOBJECT_LIMIT_FACTOR,
GC_MALLOC_LIMIT_MIN,
GC_MALLOC_LIMIT_MAX,
GC_MALLOC_LIMIT_GROWTH_FACTOR,
GC_OLDMALLOC_LIMIT_MIN,
GC_OLDMALLOC_LIMIT_MAX,
GC_OLDMALLOC_LIMIT_GROWTH_FACTOR,
};
/* GC_DEBUG:
* enable to embed GC debugging information.
*/
#ifndef GC_DEBUG
#define GC_DEBUG 0
#endif
/* RGENGC_DEBUG:
* 1: basic information
* 2: remember set operation
* 3: mark
* 4:
* 5: sweep
*/
#ifndef RGENGC_DEBUG
#ifdef RUBY_DEVEL
#define RGENGC_DEBUG -1
#else
#define RGENGC_DEBUG 0
#endif
#endif
#if RGENGC_DEBUG < 0 && !defined(_MSC_VER)
# define RGENGC_DEBUG_ENABLED(level) (-(RGENGC_DEBUG) >= (level) && ruby_rgengc_debug >= (level))
#else
# define RGENGC_DEBUG_ENABLED(level) ((RGENGC_DEBUG) >= (level))
#endif
int ruby_rgengc_debug;
/* RGENGC_PROFILE
* 0: disable RGenGC profiling
* 1: enable profiling for basic information
* 2: enable profiling for each types
*/
#ifndef RGENGC_PROFILE
# define RGENGC_PROFILE 0
#endif
/* RGENGC_ESTIMATE_OLDMALLOC
* Enable/disable to estimate increase size of malloc'ed size by old objects.
* If estimation exceeds threshold, then will invoke full GC.
* 0: disable estimation.
* 1: enable estimation.
*/
#ifndef RGENGC_ESTIMATE_OLDMALLOC
# define RGENGC_ESTIMATE_OLDMALLOC 1
#endif
#ifndef GC_PROFILE_MORE_DETAIL
# define GC_PROFILE_MORE_DETAIL 0
#endif
#ifndef GC_PROFILE_DETAIL_MEMORY
# define GC_PROFILE_DETAIL_MEMORY 0
#endif
#ifndef GC_ENABLE_LAZY_SWEEP
# define GC_ENABLE_LAZY_SWEEP 1
#endif
#ifndef VERIFY_FREE_SIZE
#if RUBY_DEBUG
#define VERIFY_FREE_SIZE 1
#else
#define VERIFY_FREE_SIZE 0
#endif
#endif
#if VERIFY_FREE_SIZE
#undef CALC_EXACT_MALLOC_SIZE
#define CALC_EXACT_MALLOC_SIZE 1
#endif
#ifndef CALC_EXACT_MALLOC_SIZE
# define CALC_EXACT_MALLOC_SIZE 0
#endif
#if defined(HAVE_MALLOC_USABLE_SIZE) || CALC_EXACT_MALLOC_SIZE > 0
# ifndef MALLOC_ALLOCATED_SIZE
# define MALLOC_ALLOCATED_SIZE 0
# endif
#else
# define MALLOC_ALLOCATED_SIZE 0
#endif
#ifndef MALLOC_ALLOCATED_SIZE_CHECK
# define MALLOC_ALLOCATED_SIZE_CHECK 0
#endif
#ifndef GC_DEBUG_STRESS_TO_CLASS
# define GC_DEBUG_STRESS_TO_CLASS RUBY_DEBUG
#endif
typedef enum {
GPR_FLAG_NONE = 0x000,
/* major reason */
GPR_FLAG_MAJOR_BY_NOFREE = 0x001,
GPR_FLAG_MAJOR_BY_OLDGEN = 0x002,
GPR_FLAG_MAJOR_BY_SHADY = 0x004,
GPR_FLAG_MAJOR_BY_FORCE = 0x008,
#if RGENGC_ESTIMATE_OLDMALLOC
GPR_FLAG_MAJOR_BY_OLDMALLOC = 0x020,
#endif
GPR_FLAG_MAJOR_MASK = 0x0ff,
/* gc reason */
GPR_FLAG_NEWOBJ = 0x100,
GPR_FLAG_MALLOC = 0x200,
GPR_FLAG_METHOD = 0x400,
GPR_FLAG_CAPI = 0x800,
GPR_FLAG_STRESS = 0x1000,
/* others */
GPR_FLAG_IMMEDIATE_SWEEP = 0x2000,
GPR_FLAG_HAVE_FINALIZE = 0x4000,
GPR_FLAG_IMMEDIATE_MARK = 0x8000,
GPR_FLAG_FULL_MARK = 0x10000,
GPR_FLAG_COMPACT = 0x20000,
GPR_FLAG_GLOBAL = 0x40000,
GPR_DEFAULT_REASON =
(GPR_FLAG_FULL_MARK | GPR_FLAG_IMMEDIATE_MARK |
GPR_FLAG_IMMEDIATE_SWEEP | GPR_FLAG_CAPI),
} gc_profile_record_flag;
typedef struct gc_profile_record {
unsigned int flags;
size_t sequence;
double gc_time;
double gc_invoke_time;
rb_hrtime_t gc_wall_time;
rb_hrtime_t gc_invoke_wall_time;
rb_hrtime_t gc_pause_time;
rb_hrtime_t gc_stop_time;
rb_hrtime_t gc_stw_time;
rb_hrtime_t gc_mark_wall_time;
rb_hrtime_t gc_sweep_wall_time;
rb_hrtime_t gc_compact_wall_time;
size_t heap_total_objects;
size_t heap_use_size;
size_t heap_total_size;
size_t moved_objects;
#if GC_PROFILE_MORE_DETAIL
double gc_mark_time;
double gc_sweep_time;
size_t heap_use_pages;
size_t heap_live_objects;
size_t heap_free_objects;
size_t allocate_increase;
size_t allocate_limit;
double prepare_time;
size_t removing_objects;
size_t empty_objects;
#if GC_PROFILE_DETAIL_MEMORY
long maxrss;
long minflt;
long majflt;
#endif
#endif
#if MALLOC_ALLOCATED_SIZE
size_t allocated_size;
#endif
#if RGENGC_PROFILE > 0
size_t old_objects;
size_t remembered_normal_objects;
size_t remembered_shady_objects;
#endif
} gc_profile_record;
struct RMoved {
VALUE flags;
VALUE dummy;
VALUE destination;
};
#define RMOVED(obj) ((struct RMoved *)(obj))
typedef uintptr_t bits_t;
enum {
BITS_SIZE = sizeof(bits_t),
BITS_BITLENGTH = ( BITS_SIZE * CHAR_BIT )
};
struct heap_page_header {
struct heap_page *page;
};
struct heap_page_body {
struct heap_page_header header;
/* char gap[]; */
/* RVALUE values[]; */
};
#define STACK_CHUNK_SIZE 500
typedef struct stack_chunk {
VALUE data[STACK_CHUNK_SIZE];
struct stack_chunk *next;
} stack_chunk_t;
typedef struct mark_stack {
stack_chunk_t *chunk;
stack_chunk_t *cache;
int index;
int limit;
size_t cache_size;
size_t unused_cache_size;
} mark_stack_t;
typedef int (*gc_compact_compare_func)(const void *l, const void *r, void *d);
typedef struct rb_heap_newobj {
uintptr_t alloc_cursor;
uintptr_t alloc_cursor_end;
struct free_region *alloc_next_region;
struct heap_page *alloc_using_page;
} rb_heap_newobj_t;
typedef struct rb_heap_struct {
short slot_size;
/* Basic statistics */
size_t total_allocated_pages;
size_t force_major_gc_count;
size_t force_incremental_marking_finish_count;
size_t total_allocated_objects;
size_t total_freed_objects;
size_t final_slots_count;
/* Sweeping statistics */
size_t freed_slots;
size_t empty_slots;
/* Bump-pointer allocation state; only this objspace's owner thread writes it. */
rb_heap_newobj_t newobj;
struct heap_page *free_pages;
struct ccan_list_head pages;
struct heap_page *sweeping_page; /* iterator for .pages */
struct heap_page *compact_cursor;
uintptr_t compact_cursor_index;
struct heap_page *pooled_pages;
size_t total_pages; /* total page count in a heap */
size_t total_slots; /* total slot count */
} rb_heap_t;
enum {
gc_stress_no_major,
gc_stress_no_immediate_sweep,
gc_stress_full_mark_after_malloc,
gc_stress_max
};
enum gc_mode {
gc_mode_none,
gc_mode_marking,
gc_mode_sweeping,
gc_mode_compacting,
};
typedef rbimpl_atomic_uint64_t gc_counter_t;
#if !defined(HAVE_GCC_ATOMIC_BUILTINS_64) && !defined(_WIN32) && \
!(defined(__sun) && defined(HAVE_ATOMIC_H) && (defined(_LP64) || defined(_I32LPx)))
# define MALLOC_COUNTERS_NEED_LOCK 1
#endif
struct gc_malloc_bytes {
gc_counter_t malloc;
gc_counter_t free;
/* Baselines the increase is measured from: malloc snapshotted at GC start
* (gc_reset_malloc_info), free re-snapshotted at gc_sweep_finish so the
* sweep's own frees never count. */
gc_counter_t malloc_at_last_gc;
gc_counter_t free_at_last_gc;
};
/* -- Deferred free of non-thread-safe T_DATA --
*
* A dead T_DATA not RUBY_TYPED_THREAD_SAFE_FREE cannot have its dfree run during a parallel
* local sweep: the dfree is extension code that may touch process state other Ractors are using,
* so it needs the world stopped (not merely serialization against other dfrees). The sweep
* therefore copies out what the free needs, reclaims the slot immediately, and the dfrees are
* called later under a VM barrier. Such a type is never embedded, so the payload always outlives
* the slot.
*/
#define TDATA_UNSAFE_FREE_CHUNK_CAPA 32
/* Drained chunks kept for reuse; the rest are freed. */
#define TDATA_UNSAFE_FREE_CACHE_MAX 64
struct tdata_unsafe_free_entry {
void (*dfree)(void *);
void *data;
};
struct tdata_unsafe_free_chunk {
struct tdata_unsafe_free_chunk *next;
unsigned int count;
uint32_t embed_xfree_bits;
struct tdata_unsafe_free_entry entries[TDATA_UNSAFE_FREE_CHUNK_CAPA];
};
STATIC_ASSERT(tdata_unsafe_free_bits_cover_chunk,
TDATA_UNSAFE_FREE_CHUNK_CAPA <= 32);
struct gc_process_stat_snapshot {
uint64_t minor_gc_count;
uint64_t major_gc_count;
uint64_t global_gc_invocation_count;
uint64_t marking_time_ns;
uint64_t sweeping_time_ns;
};
struct gc_process_stat_total {
uint64_t minor_gc_count;
uint64_t major_gc_count;
uint64_t global_gc_invocation_count;
uint64_t marking_time_ns;
uint64_t sweeping_time_ns;
};
typedef struct rb_objspace {
struct {
struct gc_malloc_bytes counters;
#if RGENGC_ESTIMATE_OLDMALLOC
size_t oldmalloc_increase;
gc_counter_t freed_in_gc;
#endif
#ifdef MALLOC_COUNTERS_NEED_LOCK
rb_nativethread_lock_t lock;
#endif
} malloc_counters;
struct {
size_t limit;
#if MALLOC_ALLOCATED_SIZE
size_t allocated_size;
size_t allocations;
#endif
} malloc_params;
struct rb_gc_config {
bool full_mark;
} gc_config;
struct {
unsigned int mode : 2;
unsigned int immediate_sweep : 1;
unsigned int dont_gc : 1;
/* A user hold from GC.disable (kept in vm->gc.disable_holders); owner thread only. */
unsigned int user_gc_disabled : 1;
unsigned int dont_incremental : 1;
unsigned int during_gc : 1;
unsigned int during_global_gc : 1;
unsigned int during_compacting : 1;
unsigned int gc_lock_barrier : 1;
unsigned int during_reference_updating : 1;
unsigned int during_minor_gc : 1;
unsigned int during_postmortem : 1;
/* No Ractor will own this objspace any more and it will never allocate again. */
unsigned int ownerless : 1;
unsigned int measure_gc : 1;
} flags;
rb_event_flag_t hook_events;
rb_heap_t heaps[HEAP_COUNT];
size_t empty_pages_count;
struct heap_page *empty_pages;
/* Nonzero while gc.c parks this objspace in its zombie table: the pages it adds to
* global_objspace->zombie_total_pages. Written under the VM lock or the barrier. */
size_t zombie_pages;
struct {
rb_atomic_t finalizing;
} atomic_flags;
mark_stack_t mark_stack;
size_t marked_slots;
/* Moved out of the per-Ractor newobj cache: allocation state is per objspace. */
size_t incremental_mark_step_allocated_slots;
/* Inputs of the global GC trigger, all owned by this objspace's thread.
* shareable_objects is the live population of shareable objects; exceeding the
* limit requests a global GC. */
size_t shareable_objects;
size_t shareable_objects_limit;
/* Whether the last mark ran the pinned walk; the sweep asserts on it. */
unsigned char last_cycle_pinned;
struct {
rb_darray(struct heap_page *) sorted;
size_t allocated_pages;
size_t freed_pages;
uintptr_t range[2];
size_t freeable_pages;
size_t allocatable_bytes;
/* final */
VALUE deferred_final;
} heap_pages;
st_table *finalizer_table;
struct {
int run;
unsigned int latest_gc_info;
gc_profile_record *records;
gc_profile_record *current_record;
size_t next_index;
size_t size;
size_t record_count;
size_t max_records;
size_t record_sequence;
#if GC_PROFILE_MORE_DETAIL
double prepare_time;
#endif
double invoke_time;
rb_hrtime_t invoke_wall_time;
size_t minor_gc_count;
size_t major_gc_count;
size_t global_gc_count;
size_t global_gc_invocation_count;
size_t compact_count;
size_t read_barrier_faults;
#if RGENGC_PROFILE > 0
size_t total_generated_normal_object_count;
size_t total_generated_shady_object_count;
size_t total_shade_operation_count;
size_t total_promoted_count;
size_t total_remembered_normal_object_count;
size_t total_remembered_shady_object_count;
#if RGENGC_PROFILE >= 2
size_t generated_normal_object_count_types[RUBY_T_MASK];
size_t generated_shady_object_count_types[RUBY_T_MASK];
size_t shade_operation_count_types[RUBY_T_MASK];
size_t promoted_types[RUBY_T_MASK];
size_t remembered_normal_object_count_types[RUBY_T_MASK];
size_t remembered_shady_object_count_types[RUBY_T_MASK];
#endif
#endif /* RGENGC_PROFILE */
/* temporary profiling space */
double gc_sweep_start_time;
rb_hrtime_t gc_wall_start_time;
rb_hrtime_t gc_sweep_wall_start_time;
rb_hrtime_t gc_sweep_excluded_wall_time;
rb_hrtime_t gc_pause_start_time;
rb_hrtime_t gc_stw_start_time;
rb_hrtime_t gc_stop_time;
rb_hrtime_t gc_mark_phase_wall_start_time;
rb_hrtime_t gc_sweep_phase_wall_start_time;
#if GC_PROFILE_MORE_DETAIL
size_t total_allocated_objects_at_gc_start;
size_t heap_used_at_gc_start;
#endif
/* basic statistics */
size_t count;
unsigned long long marking_time_ns;
struct timespec marking_start_time;
unsigned long long sweeping_time_ns;
struct timespec sweeping_start_time;
/* Weak references */
size_t weak_references_count;
} profile;
struct {
bool parent_object_old_p;
VALUE parent_object;
int need_major_gc;
size_t last_major_gc;
size_t uncollectible_wb_unprotected_objects;
size_t uncollectible_wb_unprotected_objects_limit;
size_t old_objects;
size_t old_objects_limit;
#if RGENGC_ESTIMATE_OLDMALLOC
size_t oldmalloc_increase_limit;
#endif
#if RGENGC_CHECK_MODE >= 2
struct st_table *allrefs_table;
size_t error_count;
#endif
} rgengc;
struct {
size_t considered_count_table[T_MASK];
size_t moved_count_table[T_MASK];
size_t moved_up_count_table[T_MASK];
size_t moved_down_count_table[T_MASK];
size_t total_moved;
/* This function will be used, if set, to sort the heap prior to compaction */
gc_compact_compare_func compare_func;
} rcompactor;
struct {
size_t pooled_slots;
size_t step_slots;
} rincgc;
#if GC_DEBUG_STRESS_TO_CLASS
VALUE stress_to_class;
#endif
rb_darray(VALUE) weak_references;
rb_postponed_job_handle_t finalize_deferred_pjob;
/* Partially filled chunk of deferred non-thread-safe T_DATA metadata. */
struct tdata_unsafe_free_chunk *tdata_unsafe_free_chunk;
int sweeping_heap_count;
int fork_vm_lock_lev;
struct rb_gc_vm_context vm_context;
/* Process-wide GC statistics publication. Default GC only: other
* implementations reject GC.stat(process: true) and never initialize
* this lock. */
struct {
rb_nativethread_lock_t lock;
struct gc_process_stat_snapshot published;
} process_stat;
/* The incremental-marking flag as an addressable field, so that ZJIT-generated
* code can read it; see rb_gc_impl_zjit_writebarrier_required_offset.
* Single-writer: a local GC runs on the owning Ractor's thread and a global GC
* writes under the stop-the-world barrier. Read by JIT code on the same thread. */
bool zjit_during_incremental_marking;
} rb_objspace_t;
/* The one VM-global GC structure; for now it only holds the page pool. Page bodies are
* carved out of large mmap arenas and reused via a process-wide freelist (per-page
* mmap/munmap would serialize on the kernel's mmap_lock). Leaf lock: no alloc, no GC. */
typedef struct rb_global_objspace {
struct {
rb_nativethread_lock_t lock;
struct heap_page_body *hot_list; /* ≤ PAGE_POOL_HOT_MAX un-advised bodies; link at body offset 0 */
int hot_count;
size_t os_page_size; /* sysconf(_SC_PAGE_SIZE), cached at init */
/* List of mmap'd memory regions (arenas) for page bodies. */
struct page_arena {
struct page_arena *next;
char *start; /* usable area, HEAP_PAGE_ALIGN aligned */
size_t size; /* usable bytes (a multiple of HEAP_PAGE_SIZE) */
struct heap_page_body *cold_freelist; /* free bodies of this arena; link at body offset 0 */
int free_count; /* bodies of this arena currently free (hot list + cold_freelist) */
int cold_count; /* bodies on cold_freelist (subset of free_count) */
} *arenas; /* every arena, newest first */
char *arena_cursor; /* first body not yet carved out of the newest arena */
char *arena_end; /* end of current arena */
struct page_arena *arena_current; /* arena that arena_cursor carves from */
int arena_count; /* current mapped arenas (for GC.stat total_pages) */
int advised_count; /* page bodies with MADV_* applied (for GC.stat discarded pages) */
size_t arenas_unmapped; /* cumulative arenas munmapped (for GC.stat) */
} page_pool;
/* Pages held by zombie objspaces (sum of their zombie_pages), and what was left of
* it after the last global cycle (roughly the live data). Written under the VM
* lock or the barrier; readers (gc_need_global_p) may be racy. */
size_t zombie_total_pages;
size_t zombie_pages_survivors;
/* An objspace merge (objspace_absorb) is running: suppress the cross-objspace
* verifier while the graph is in flux. Written by the absorbing thread, read by
* verification with the world stopped. */
bool during_absorb;
/* main's objspace, for gc_enter's locking policy. main_ractor->objspace is swapped
* during Ractor creation; this stable pointer decides the same way at both ends of a
* GC. Set at boot, re-pointed in a forked child. */
rb_objspace_t *main_objspace;
/* GC.stress is process-global (upstream semantics). Written by GC.stress= in any
* Ractor (rare) and read on every Ractor's alloc and GC path; it is diagnostic, so
* plain store/load with last-writer-wins is fine. */
bool gc_stressful;
VALUE gc_stress_mode;
/* Global GC driver state. compacting is true during the move phase: reference
* updates are deferred until all forwarding exists, so a cross-objspace reference is
* rewritten exactly once. objspaces is the snapshot being collected. */
struct {
bool compacting;
struct rb_objspace **objspaces;
size_t n_objspaces, objspaces_capa;
size_t count;
} global_gc;
/* Index of every objspace's heap pages, ordered by body address. Writers (page
* alloc/free) serialize on page_pool.lock; the only reader is a stop-the-world global
* GC, so reads need no lock. A local GC uses its own heap_pages.sorted. */
struct {
struct heap_page **pages;
size_t n_pages, capa;
uintptr_t lomem, himem;
} page_index;
rb_postponed_job_handle_t tdata_deferred_free_pjob; /* atomic */
/* Pending count of deferred non-thread-safe T_DATA frees across all objspaces:
* bumped as each one is deferred, reset to 0 by the drain. Crossing
* TDATA_DEFERRED_FREE_THRESHOLD triggers the postponed job. */
size_t tdata_deferred_free_count; /* atomic */
/* Full chunks awaiting a drain (CAS stack), and drained chunks kept for reuse. */
struct tdata_unsafe_free_chunk *tdata_unsafe_free_published; /* atomic */
struct tdata_unsafe_free_chunk *tdata_unsafe_free_cache; /* atomic */
size_t tdata_unsafe_free_cache_len; /* atomic */
/* Archive of destroyed objspaces' final statistics, added once on absorption. */
struct gc_process_stat_total process_stat_archive;
} rb_global_objspace_t;
static rb_global_objspace_t rb_global_objspace_instance;
static rb_global_objspace_t *global_objspace = NULL;
/* Relaxed: every reader only asks whether a drain is worth arranging, and the drain
* itself stops the world before it touches a chunk. */
static inline size_t
tdata_deferred_free_count_load(void)
{
return rbimpl_atomic_size_load(&global_objspace->tdata_deferred_free_count, RBIMPL_ATOMIC_RELAXED);
}
/* The floor keeps a global GC from running as soon as a few shareable objects appear;
* the factor follows the rule used for the old-generation limit. */
#define SHAREABLE_OBJECTS_LIMIT_MIN (1 << 16)
#define SHAREABLE_OBJECTS_LIMIT_FACTOR 2.0
/* Start a global GC once terminated, uninherited Ractors hold this many heap pages. A
* small Ractor's objspace is about 13 pages, so discarding many of them still stays
* below it, while a single fat zombie crosses it. */
#define ZOMBIE_PAGES_TRIGGER 256
/* Trigger the deferred T_DATA free postponed job once this many have accumulated
* across all objspaces. */
#define TDATA_DEFERRED_FREE_THRESHOLD (1 << 15)
static void objspace_absorb(rb_objspace_t *dst, rb_objspace_t *src);
static struct heap_page_body *page_pool_acquire(struct page_arena **arena_out);
static void page_pool_release(struct heap_page_body *body, struct page_arena *arena);
#ifdef HAVE_MMAP
static void page_pool_release_locked(struct heap_page_body *body, struct page_arena *arena);
#endif
static void page_pool_reclaim(rb_global_objspace_t *g);
#if RGENGC_CHECK_MODE && !defined(_WIN32) && !defined(__wasi__) && defined(HAVE_PTHREAD_H)
# define PAGE_POOL_LOCK_ERRORCHECK 1
#endif
static void
page_pool_lock_initialize(rb_nativethread_lock_t *lock)
{
#ifdef PAGE_POOL_LOCK_ERRORCHECK
pthread_mutexattr_t attr;
pthread_mutexattr_init(&attr);
pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_ERRORCHECK);
pthread_mutex_init(lock, &attr);
pthread_mutexattr_destroy(&attr);
#else
rb_native_mutex_initialize(lock);
#endif
}
static void
global_objspace_init(void)
{
if (global_objspace == NULL) {
rb_global_objspace_t *g = &rb_global_objspace_instance;
page_pool_lock_initialize(&g->page_pool.lock);
g->page_pool.hot_list = NULL;
g->page_pool.hot_count = 0;
g->page_pool.arenas = NULL;
g->page_pool.arena_cursor = NULL;
g->page_pool.arena_end = NULL;
g->page_pool.arena_current = NULL;
g->page_pool.arena_count = 0;
g->page_pool.advised_count = 0;
g->tdata_deferred_free_pjob = POSTPONED_JOB_HANDLE_INVALID;
g->tdata_unsafe_free_published = NULL;
g->tdata_unsafe_free_cache = NULL;
g->tdata_unsafe_free_cache_len = 0;
g->page_pool.arenas_unmapped = 0;
#ifdef HAVE_MMAP
g->page_pool.os_page_size = sysconf(_SC_PAGE_SIZE);
#else
g->page_pool.os_page_size = 0;
#endif
global_objspace = g;
}
}
#ifndef HEAP_PAGE_ALIGN_LOG
/* default tiny heap size: 64KiB */
#define HEAP_PAGE_ALIGN_LOG 16
#endif
#if GC_DEBUG
struct rvalue_overhead {
const char *file;
int line;
};
// Make sure that RVALUE_OVERHEAD aligns to sizeof(VALUE)
# define RVALUE_OVERHEAD (sizeof(struct { \
union { \
struct rvalue_overhead overhead; \
VALUE value; \
}; \
}))
size_t rb_gc_impl_obj_slot_size(VALUE obj);
# define GET_RVALUE_OVERHEAD(obj) ((struct rvalue_overhead *)((uintptr_t)obj + rb_gc_impl_obj_slot_size(obj)))
#else
# ifndef RVALUE_OVERHEAD
# define RVALUE_OVERHEAD 0
# endif
#endif
#define RVALUE_SLOT_SIZE (sizeof(struct RBasic) + sizeof(VALUE[RBIMPL_RVALUE_EMBED_LEN_MAX]) + RVALUE_OVERHEAD)
static const size_t pool_slot_sizes[HEAP_COUNT] = {
#define SLOT(size) ((size) + RVALUE_OVERHEAD),
EACH_POOL_SLOT_SIZE(SLOT)
#undef SLOT
};
/* An init size below one slot in the largest heap never forces that heap's first
* page, and allocating there then fails with "cannot create a new page after GC". */
static inline size_t
heap_init_bytes_min(void)
{
return pool_slot_sizes[HEAP_COUNT - 1];
}
/* Precomputed reciprocals for fast slot index calculation.
* For slot size d: reciprocal = ceil(2^48 / d).
* Then offset / d == (uint32_t)((offset * reciprocal) >> 48)
* for all offset < HEAP_PAGE_SIZE. */
#define SLOT_RECIPROCAL_SHIFT 48
#define SLOT_RECIPROCAL(size) (((1ULL << SLOT_RECIPROCAL_SHIFT) + (size) - 1) / (size))
static const uint64_t heap_slot_reciprocal_table[HEAP_COUNT] = {
#define SLOT(size) SLOT_RECIPROCAL((size) + RVALUE_OVERHEAD),
EACH_POOL_SLOT_SIZE(SLOT)
#undef SLOT
};
#if SIZEOF_VALUE >= 8
static uint8_t size_to_heap_idx[1024 / 8 + 1];
#else
static uint8_t size_to_heap_idx[512 / 8 + 1];
#endif
#ifndef MAX
# define MAX(a, b) (((a) > (b)) ? (a) : (b))
#endif
#ifndef MIN
# define MIN(a, b) (((a) < (b)) ? (a) : (b))
#endif
#define roomof(x, y) (((x) + (y) - 1) / (y))
#define CEILDIV(i, mod) roomof(i, mod)
#define MIN_POOL_SLOT_SIZE 32
enum {
HEAP_PAGE_ALIGN = (1UL << HEAP_PAGE_ALIGN_LOG),
HEAP_PAGE_ALIGN_MASK = (~(~0UL << HEAP_PAGE_ALIGN_LOG)),
HEAP_PAGE_SIZE = HEAP_PAGE_ALIGN,
HEAP_PAGE_BITMAP_LIMIT = CEILDIV(CEILDIV(HEAP_PAGE_SIZE, MIN_POOL_SLOT_SIZE), BITS_BITLENGTH),
HEAP_PAGE_BITMAP_SIZE = (BITS_SIZE * HEAP_PAGE_BITMAP_LIMIT),
};
#define HEAP_PAGE_ALIGN (1 << HEAP_PAGE_ALIGN_LOG)
#define HEAP_PAGE_SIZE HEAP_PAGE_ALIGN
#if !defined(INCREMENTAL_MARK_STEP_ALLOCATIONS)
# define INCREMENTAL_MARK_STEP_ALLOCATIONS 500
#endif
#undef INIT_HEAP_PAGE_ALLOC_USE_MMAP
/* Must define either HEAP_PAGE_ALLOC_USE_MMAP or
* INIT_HEAP_PAGE_ALLOC_USE_MMAP. */
#ifndef HAVE_MMAP
/* We can't use mmap of course, if it is not available. */
static const bool HEAP_PAGE_ALLOC_USE_MMAP = false;
#elif defined(__wasm__)
/* wasmtime does not have proper support for mmap.
* See https://github.com/bytecodealliance/wasmtime/blob/main/docs/WASI-rationale.md#why-no-mmap-and-friends
*/
static const bool HEAP_PAGE_ALLOC_USE_MMAP = false;
#elif HAVE_CONST_PAGE_SIZE
/* If we have the PAGE_SIZE and it is a constant, then we can directly use it. */
static const bool HEAP_PAGE_ALLOC_USE_MMAP = (PAGE_SIZE <= HEAP_PAGE_SIZE);
#elif defined(PAGE_MAX_SIZE) && (PAGE_MAX_SIZE <= HEAP_PAGE_SIZE)
/* If we can use the maximum page size. */
static const bool HEAP_PAGE_ALLOC_USE_MMAP = true;
#elif defined(PAGE_SIZE)
/* If the PAGE_SIZE macro can be used dynamically. */
# define INIT_HEAP_PAGE_ALLOC_USE_MMAP (PAGE_SIZE <= HEAP_PAGE_SIZE)
#elif defined(HAVE_SYSCONF) && defined(_SC_PAGE_SIZE)
/* If we can use sysconf to determine the page size. */
# define INIT_HEAP_PAGE_ALLOC_USE_MMAP (sysconf(_SC_PAGE_SIZE) <= HEAP_PAGE_SIZE)
#else
/* Otherwise we can't determine the system page size, so don't use mmap. */
static const bool HEAP_PAGE_ALLOC_USE_MMAP = false;
#endif
#ifdef INIT_HEAP_PAGE_ALLOC_USE_MMAP
/* We can determine the system page size at runtime. */
# define HEAP_PAGE_ALLOC_USE_MMAP (heap_page_alloc_use_mmap != false)
static bool heap_page_alloc_use_mmap;
#endif
#define RVALUE_AGE_BIT_COUNT 2
#define RVALUE_AGE_BIT_MASK (((bits_t)1 << RVALUE_AGE_BIT_COUNT) - 1)
#define RVALUE_OLD_AGE 3
struct free_region {
VALUE flags; /* always 0 for freed obj */
uintptr_t end; /* exclusive end address of the run */
struct free_region *next; /* next free region in the page */
};
struct heap_page {
/* Cache line 0: allocation fast path + SLOT_INDEX */
struct free_region *free_region;
uintptr_t start;
uint64_t slot_size_reciprocal;
unsigned short slot_size;
unsigned short total_slots;
unsigned short free_slots;
unsigned short final_slots;
unsigned short pinned_slots;
/* Page state flags. A bitfield is safe: the only writers are the owning Ractor
* (GVL) and the global GC driver (stop-the-world), never together. has_shareable /
* has_shref hint that the page holds at least one such bit, for re-scanning. */
struct {
unsigned int before_sweep : 1;
unsigned int has_remembered_objects : 1;
unsigned int has_uncollectible_wb_unprotected_objects : 1;
unsigned int has_shref_objects : 1;
unsigned int has_shareable_objects : 1;
} flags;
rb_heap_t *heap;
/* The objspace owning this page, so any object's owner is one load away
* (GET_HEAP_OBJSPACE). Rewritten only when a page changes owner (inheritance). */
rb_objspace_t *objspace;
struct heap_page *free_next;
struct heap_page_body *body;
struct page_arena *arena;
struct ccan_list_node page_node;
bits_t wb_unprotected_bits[HEAP_PAGE_BITMAP_LIMIT];
/* the following three bitmaps are cleared at the beginning of full GC */
bits_t mark_bits[HEAP_PAGE_BITMAP_LIMIT];
bits_t uncollectible_bits[HEAP_PAGE_BITMAP_LIMIT];
bits_t marking_bits[HEAP_PAGE_BITMAP_LIMIT];
bits_t remembered_bits[HEAP_PAGE_BITMAP_LIMIT];
/* Two extra bits per object. shareable_bits: what a local sweep must never free
* (only a global GC decides a shareable object is dead); set at creation and by
* rb_gc_impl_obj_became_shareable. shref_bits: an unshareable object referenced
* from a shareable one, a local GC root; the write barrier maintains it. */
bits_t shareable_bits[HEAP_PAGE_BITMAP_LIMIT];
bits_t shref_bits[HEAP_PAGE_BITMAP_LIMIT];
/* If set, the object is not movable */
bits_t pinned_bits[HEAP_PAGE_BITMAP_LIMIT];
bits_t age_bits[HEAP_PAGE_BITMAP_LIMIT * RVALUE_AGE_BIT_COUNT];
};
/*
* When asan is enabled, this will prohibit writing to the freelist until it is unlocked
*/
static void
asan_lock_freelist(struct heap_page *page)
{
asan_poison_memory_region(&page->free_region, sizeof(struct free_region *));
}
/*
* When asan is enabled, this will enable the ability to write to the freelist
*/
static void
asan_unlock_freelist(struct heap_page *page)
{
asan_unpoison_memory_region(&page->free_region, sizeof(struct free_region *), false);
}
static inline bool
heap_page_in_global_empty_pages_pool(rb_objspace_t *objspace, struct heap_page *page)
{
if (page->total_slots == 0) {
GC_ASSERT(page->start == 0);
GC_ASSERT(page->slot_size == 0);
GC_ASSERT(page->heap == NULL);
GC_ASSERT(page->free_slots == 0);
asan_unpoisoning_memory_region(&page->free_region, sizeof(&page->free_region)) {
GC_ASSERT(page->free_region == NULL);
}
return true;
}
else {
GC_ASSERT(page->start != 0);
GC_ASSERT(page->slot_size != 0);
GC_ASSERT(page->heap != NULL);
return false;
}
}
#define GET_PAGE_BODY(x) ((struct heap_page_body *)((bits_t)(x) & ~(HEAP_PAGE_ALIGN_MASK)))
#define GET_PAGE_HEADER(x) (&GET_PAGE_BODY(x)->header)
#define GET_HEAP_PAGE(x) (GET_PAGE_HEADER(x)->page)
static inline size_t
slot_index_for_offset(size_t offset, uint64_t reciprocal)
{
return (uint32_t)(((uint64_t)offset * reciprocal) >> SLOT_RECIPROCAL_SHIFT);
}
#define SLOT_INDEX(page, p) slot_index_for_offset((uintptr_t)(p) - (page)->start, (page)->slot_size_reciprocal)
#define SLOT_BITMAP_INDEX(page, p) (SLOT_INDEX(page, p) / BITS_BITLENGTH)
#define SLOT_BITMAP_OFFSET(page, p) (SLOT_INDEX(page, p) & (BITS_BITLENGTH - 1))
#define SLOT_BITMAP_BIT(page, p) ((bits_t)1 << SLOT_BITMAP_OFFSET(page, p))
#define _MARKED_IN_BITMAP(bits, page, p) ((bits)[SLOT_BITMAP_INDEX(page, p)] & SLOT_BITMAP_BIT(page, p))
#define _MARK_IN_BITMAP(bits, page, p) ((bits)[SLOT_BITMAP_INDEX(page, p)] |= SLOT_BITMAP_BIT(page, p))
#define _CLEAR_IN_BITMAP(bits, page, p) ((bits)[SLOT_BITMAP_INDEX(page, p)] &= ~SLOT_BITMAP_BIT(page, p))
#define MARKED_IN_BITMAP(bits, p) _MARKED_IN_BITMAP(bits, GET_HEAP_PAGE(p), p)
#define MARK_IN_BITMAP(bits, p) _MARK_IN_BITMAP(bits, GET_HEAP_PAGE(p), p)
#define CLEAR_IN_BITMAP(bits, p) _CLEAR_IN_BITMAP(bits, GET_HEAP_PAGE(p), p)
#define GET_HEAP_MARK_BITS(x) (&GET_HEAP_PAGE(x)->mark_bits[0])
#define GET_HEAP_PINNED_BITS(x) (&GET_HEAP_PAGE(x)->pinned_bits[0])
#define GET_HEAP_UNCOLLECTIBLE_BITS(x) (&GET_HEAP_PAGE(x)->uncollectible_bits[0])
#define GET_HEAP_WB_UNPROTECTED_BITS(x) (&GET_HEAP_PAGE(x)->wb_unprotected_bits[0])
#define GET_HEAP_MARKING_BITS(x) (&GET_HEAP_PAGE(x)->marking_bits[0])
#define GET_HEAP_SHAREABLE_BITS(x) (&GET_HEAP_PAGE(x)->shareable_bits[0])
#define GET_HEAP_SHREF_BITS(x) (&GET_HEAP_PAGE(x)->shref_bits[0])
#define GET_HEAP_OBJSPACE(x) (GET_HEAP_PAGE(x)->objspace)
/* obj lives on a page of another objspace, not the current one (i.e. it is foreign). */
static inline bool
gc_foreign_object_p(const rb_objspace_t *objspace, VALUE obj)
{
if (RB_LIKELY(!rb_gc_ever_multi_ractor_p())) return false;
return RB_UNLIKELY(GET_HEAP_OBJSPACE(obj) != objspace);
}
/* Foreign and not inside a stop-the-world global GC. While true, a local GC must not
* touch obj's per-object GC state (mark, pin, remember bits): its owner handles that,
* or the global GC does with everyone stopped. */
static inline bool
gc_skip_foreign_object_p(const rb_objspace_t *objspace, VALUE obj)
{
return gc_foreign_object_p(objspace, obj) && !objspace->flags.during_global_gc;
}
/* Record obj as shareable on its owning page (bit, page flag and population counter).
* Shared by born-shareable objects and make_shareable. The writer is the owner thread,
* so plain bit operations suffice. */
static inline void
gc_page_add_shareable(struct heap_page *page, VALUE obj)
{
GC_ASSERT(page == GET_HEAP_PAGE(obj));
GC_ASSERT(RB_FL_TEST_RAW(obj, RUBY_FL_SHAREABLE));
_MARK_IN_BITMAP(page->shareable_bits, page, obj);
page->flags.has_shareable_objects = TRUE;
page->objspace->shareable_objects++;
}
static int
RVALUE_AGE_GET(VALUE obj)
{
struct heap_page *page = GET_HEAP_PAGE(obj);
bits_t *age_bits = page->age_bits;
size_t slot_idx = SLOT_INDEX(page, obj);
size_t idx = (slot_idx / BITS_BITLENGTH) * 2;
int shift = (int)(slot_idx & (BITS_BITLENGTH - 1));
int lo = (age_bits[idx] >> shift) & 1;
int hi = (age_bits[idx + 1] >> shift) & 1;
return lo | (hi << 1);
}
static void
RVALUE_AGE_SET_BITMAP(VALUE obj, int age)
{
RUBY_ASSERT(age <= RVALUE_OLD_AGE);
struct heap_page *page = GET_HEAP_PAGE(obj);
bits_t *age_bits = page->age_bits;
size_t slot_idx = SLOT_INDEX(page, obj);
size_t idx = (slot_idx / BITS_BITLENGTH) * 2;
int shift = (int)(slot_idx & (BITS_BITLENGTH - 1));
bits_t mask = (bits_t)1 << shift;
age_bits[idx] = (age_bits[idx] & ~mask) | ((bits_t)(age & 1) << shift);
age_bits[idx + 1] = (age_bits[idx + 1] & ~mask) | ((bits_t)((age >> 1) & 1) << shift);
}
static void
RVALUE_AGE_SET(VALUE obj, int age)
{
RVALUE_AGE_SET_BITMAP(obj, age);
if (age == RVALUE_OLD_AGE) {
RB_FL_SET_RAW(obj, RUBY_FL_PROMOTED);
}
else {
RB_FL_UNSET_RAW(obj, RUBY_FL_PROMOTED);
}
}
#define malloc_limit objspace->malloc_params.limit
#define malloc_increase gc_malloc_counters_increase_unsigned(objspace, &objspace->malloc_counters.counters)
#define malloc_allocated_size objspace->malloc_params.allocated_size
#ifdef MALLOC_COUNTERS_NEED_LOCK
# define MALLOC_COUNTERS_LOCK(o) rb_native_mutex_lock(&(o)->malloc_counters.lock)
# define MALLOC_COUNTERS_UNLOCK(o) rb_native_mutex_unlock(&(o)->malloc_counters.lock)
#else
# define MALLOC_COUNTERS_LOCK(o) ((void)0)
# define MALLOC_COUNTERS_UNLOCK(o) ((void)0)
#endif
static inline void
gc_counter_add(gc_counter_t *p, size_t delta)
{
#ifdef MALLOC_COUNTERS_NEED_LOCK
*p += (gc_counter_t)delta;
#else
rbimpl_atomic_u64_fetch_add_relaxed(p, (uint64_t)delta);
#endif
}
static inline gc_counter_t
gc_counter_load_relaxed(const gc_counter_t *p)
{
#ifdef MALLOC_COUNTERS_NEED_LOCK
return *p;
#else
return rbimpl_atomic_u64_load_relaxed(p);
#endif
}
static inline gc_counter_t
gc_counter_load_acquire(const gc_counter_t *p)
{
#ifdef MALLOC_COUNTERS_NEED_LOCK
return *p;
#else
return rbimpl_atomic_u64_load_acquire(p);
#endif
}
static inline void
gc_counter_store_release(gc_counter_t *p, gc_counter_t v)
{
#ifdef MALLOC_COUNTERS_NEED_LOCK
*p = v;
#else
rbimpl_atomic_u64_set_release(p, v);
#endif
}
static inline int64_t
gc_malloc_counters_increase(rb_objspace_t *objspace, const struct gc_malloc_bytes *c)
{
MALLOC_COUNTERS_LOCK(objspace);
gc_counter_t malloc_at = gc_counter_load_acquire(&c->malloc_at_last_gc);
gc_counter_t free_at = gc_counter_load_acquire(&c->free_at_last_gc);
gc_counter_t malloc_now = gc_counter_load_relaxed(&c->malloc);
gc_counter_t free_now = gc_counter_load_relaxed(&c->free);
MALLOC_COUNTERS_UNLOCK(objspace);
gc_counter_t malloc_delta = malloc_now - malloc_at;
gc_counter_t free_delta = free_now - free_at;
if (malloc_delta >= free_delta) {
return (int64_t)(malloc_delta - free_delta);
}
else {
return -(int64_t)(free_delta - malloc_delta);
}
}
static inline size_t
gc_malloc_counters_increase_unsigned(rb_objspace_t *objspace, const struct gc_malloc_bytes *c)
{
int64_t inc = gc_malloc_counters_increase(objspace, c);
if (inc <= 0) return 0;
#if SIZEOF_SIZE_T < 8
if ((uint64_t)inc > SIZE_MAX) return SIZE_MAX;
#endif
return (size_t)inc;
}
/* Frees done while sweeping are the GC's own work, not the mutator's: advance
* free_at_last_gc past them so they cannot pay for the next cycle's allocation.
* malloc_at_last_gc stays at gc_reset_malloc_info's snapshot (GC start). */
static inline void
gc_malloc_counters_snapshot_free_at_last_gc(rb_objspace_t *objspace, struct gc_malloc_bytes *c)
{
MALLOC_COUNTERS_LOCK(objspace);
gc_counter_store_release(&c->free_at_last_gc, gc_counter_load_relaxed(&c->free));
MALLOC_COUNTERS_UNLOCK(objspace);
}
static inline void
gc_malloc_counters_snapshot(rb_objspace_t *objspace, struct gc_malloc_bytes *c)
{
MALLOC_COUNTERS_LOCK(objspace);
gc_counter_t malloc_now = gc_counter_load_relaxed(&c->malloc);
gc_counter_t free_now = gc_counter_load_relaxed(&c->free);
gc_counter_store_release(&c->malloc_at_last_gc, malloc_now);
gc_counter_store_release(&c->free_at_last_gc, free_now);
MALLOC_COUNTERS_UNLOCK(objspace);
}
#if RGENGC_ESTIMATE_OLDMALLOC
static inline size_t
gc_oldmalloc_increase_load(rb_objspace_t *objspace)
{
return rbimpl_atomic_size_load(&objspace->malloc_counters.oldmalloc_increase, RBIMPL_ATOMIC_RELAXED);
}
#endif
#define heap_pages_lomem objspace->heap_pages.range[0]
#define heap_pages_himem objspace->heap_pages.range[1]
#define heap_pages_freeable_pages objspace->heap_pages.freeable_pages
#define heap_pages_deferred_final objspace->heap_pages.deferred_final
#define heaps objspace->heaps
#define during_gc objspace->flags.during_gc
#define finalizing objspace->atomic_flags.finalizing
#define finalizer_table objspace->finalizer_table
#define ruby_gc_stressful global_objspace->gc_stressful
#define ruby_gc_stress_mode global_objspace->gc_stress_mode
#if GC_DEBUG_STRESS_TO_CLASS
#define stress_to_class objspace->stress_to_class
#define set_stress_to_class(c) (stress_to_class = (c))
#else
#define stress_to_class ((void)objspace, 0)
#define set_stress_to_class(c) ((void)objspace, (c))
#endif
#if 0
#define dont_gc_on() (fprintf(stderr, "dont_gc_on@%s:%d\n", __FILE__, __LINE__), objspace->flags.dont_gc = 1)
#define dont_gc_off() (fprintf(stderr, "dont_gc_off@%s:%d\n", __FILE__, __LINE__), objspace->flags.dont_gc = 0)
#define dont_gc_set(b) (fprintf(stderr, "dont_gc_set(%d)@%s:%d\n", __FILE__, __LINE__), objspace->flags.dont_gc = (int)(b))
#define dont_gc_val() (objspace->flags.dont_gc)
#else
#define dont_gc_on() (objspace->flags.dont_gc = 1)
#define dont_gc_off() (objspace->flags.dont_gc = 0)
#define dont_gc_set(b) (objspace->flags.dont_gc = (int)(b))
#define dont_gc_val() (objspace->flags.dont_gc)
#endif
#define gc_config_full_mark_set(b) (objspace->gc_config.full_mark = (int)(b))
#define gc_config_full_mark_val (objspace->gc_config.full_mark)
static inline enum gc_mode
gc_mode_verify(enum gc_mode mode)
{
#if RGENGC_CHECK_MODE > 0
switch (mode) {
case gc_mode_none:
case gc_mode_marking:
case gc_mode_sweeping:
case gc_mode_compacting:
break;
default:
rb_bug("gc_mode_verify: unreachable (%d)", (int)mode);
}
#endif
return mode;
}
static inline bool
has_sweeping_pages(rb_objspace_t *objspace)
{
return objspace->sweeping_heap_count != 0;
}
static inline size_t
heap_eden_total_pages(rb_objspace_t *objspace)
{
size_t count = 0;
for (int i = 0; i < HEAP_COUNT; i++) {
count += (&heaps[i])->total_pages;
}
return count;
}
static inline size_t
total_allocated_objects(rb_objspace_t *objspace)
{
size_t count = 0;
for (int i = 0; i < HEAP_COUNT; i++) {
rb_heap_t *heap = &heaps[i];
count += heap->total_allocated_objects;
}
return count;
}
static inline size_t
total_freed_objects(rb_objspace_t *objspace)
{
size_t count = 0;
for (int i = 0; i < HEAP_COUNT; i++) {
rb_heap_t *heap = &heaps[i];
count += heap->total_freed_objects;
}
return count;
}
static inline size_t
total_final_slots_count(rb_objspace_t *objspace)
{
size_t count = 0;
for (int i = 0; i < HEAP_COUNT; i++) {
rb_heap_t *heap = &heaps[i];
count += heap->final_slots_count;
}
return count;
}
#define gc_mode(objspace) gc_mode_verify((enum gc_mode)(objspace)->flags.mode)
#define gc_mode_set(objspace, m) ((objspace)->flags.mode = (unsigned int)gc_mode_verify(m))
#define gc_needs_major_flags objspace->rgengc.need_major_gc
#define is_marking(objspace) (gc_mode(objspace) == gc_mode_marking)
#define is_sweeping(objspace) (gc_mode(objspace) == gc_mode_sweeping)
#define is_full_marking(objspace) ((objspace)->flags.during_minor_gc == FALSE)
#define is_incremental_marking(objspace) ((objspace)->zjit_during_incremental_marking != FALSE)
#define will_be_incremental_marking(objspace) ((objspace)->rgengc.need_major_gc != GPR_FLAG_NONE)
static void
gc_set_incremental_marking(rb_objspace_t *objspace, unsigned int value)
{
if (objspace->zjit_during_incremental_marking != value) {
objspace->zjit_during_incremental_marking = value;
}
}
/*
* Byte budget for incremental sweep steps. Each step sweeps at most
* this many bytes worth of slots before yielding. The effective slot
* count per step is GC_INCREMENTAL_SWEEP_BYTES / heap->slot_size,
* so larger slot pools (which are less heavily used) naturally get
* fewer slots swept per step.
*
* Baseline: 2048 slots * RVALUE_SLOT_SIZE = 2048 * 40 = 81920 bytes,
* preserving the historical behavior for the smallest heap.
*/
#define GC_INCREMENTAL_SWEEP_BYTES (2048 * RVALUE_SLOT_SIZE)
#define GC_INCREMENTAL_SWEEP_POOL_BYTES (1024 * RVALUE_SLOT_SIZE)
#define is_lazy_sweeping(objspace) (GC_ENABLE_LAZY_SWEEP && has_sweeping_pages(objspace))
/* In lazy sweeping or the previous incremental marking finished and did not yield a free page. */
#define needs_continue_sweeping(objspace, heap) \
((heap)->free_pages == NULL && is_lazy_sweeping(objspace))
#if SIZEOF_LONG == SIZEOF_VOIDP
# define obj_id_to_ref(objid) ((objid) ^ FIXNUM_FLAG) /* unset FIXNUM_FLAG */
#elif SIZEOF_LONG_LONG == SIZEOF_VOIDP
# define obj_id_to_ref(objid) (FIXNUM_P(objid) ? \
((objid) ^ FIXNUM_FLAG) : (NUM2PTR(objid) << 1))
#else
# error not supported
#endif
struct RZombie {
VALUE flags;
VALUE next;
void (*dfree)(void *);
void *data;
};
#define RZOMBIE(o) ((struct RZombie *)(o))
static bool ruby_enable_autocompact = false;
#if RGENGC_CHECK_MODE
static gc_compact_compare_func ruby_autocompact_compare_func;
#endif
static void init_mark_stack(mark_stack_t *stack);
static int garbage_collect(rb_objspace_t *, unsigned int reason);
static int gc_start(rb_objspace_t *objspace, unsigned int reason);
static void gc_rest(rb_objspace_t *objspace);
/* GC cycle events (ENTER, EXIT, START, END_MARK, END_SWEEP) fire only if the objspace's
* own Ractor enabled them, so a concurrent local GC never walks the VM-global hook list
* while another Ractor mutates it. NEWOBJ and FREEOBJ were already restricted. */
#define gc_event_hook(objspace, event) do { \
if (RB_UNLIKELY((objspace)->hook_events & (event))) { \
rb_gc_event_hook(0, (event)); \
} \
} while (0)
enum gc_enter_event {
gc_enter_event_start,
gc_enter_event_continue,
gc_enter_event_rest,
gc_enter_event_finalizer,
gc_enter_event_global,
gc_enter_event_global_auto,
};
static inline bool gc_enter(rb_objspace_t *objspace, enum gc_enter_event event, unsigned int *lock_lev);
static inline void gc_exit(rb_objspace_t *objspace, enum gc_enter_event event, unsigned int *lock_lev);
static void gc_marking_enter(rb_objspace_t *objspace);
static void gc_marking_exit(rb_objspace_t *objspace);
static void gc_sweeping_enter(rb_objspace_t *objspace);
static void gc_sweeping_exit(rb_objspace_t *objspace);
static bool gc_marks_continue(rb_objspace_t *objspace, rb_heap_t *heap);
static void gc_sweep(rb_objspace_t *objspace);
static void gc_sweep_finish_heap(rb_objspace_t *objspace, rb_heap_t *heap);
static void gc_sweep_continue(rb_objspace_t *objspace, rb_heap_t *heap);
static inline void gc_mark(rb_objspace_t *objspace, VALUE ptr);
static inline void gc_pin(rb_objspace_t *objspace, VALUE ptr);
static inline void gc_mark_and_pin(rb_objspace_t *objspace, VALUE ptr);
static int gc_mark_stacked_objects_incremental(rb_objspace_t *, size_t count);
NO_SANITIZE("memory", static inline bool is_pointer_to_heap(rb_objspace_t *objspace, const void *ptr));
static void gc_verify_internal_consistency(void *objspace_ptr);
static double getrusage_time(void);
static inline rb_hrtime_t elapsed_hrtime_from(rb_hrtime_t start);
static inline void gc_prof_setup_new_record(rb_objspace_t *objspace, unsigned int reason);
static inline void gc_prof_timer_start(rb_objspace_t *);
static inline void gc_prof_timer_stop(rb_objspace_t *);
static inline void gc_prof_mark_timer_start(rb_objspace_t *);
static inline void gc_prof_mark_timer_stop(rb_objspace_t *);
static inline void gc_prof_sweep_timer_start(rb_objspace_t *);
static inline void gc_prof_sweep_timer_stop(rb_objspace_t *);
static inline void gc_prof_set_malloc_info(rb_objspace_t *);
static inline void gc_prof_set_heap_info(rb_objspace_t *);
#define gc_prof_record(objspace) (objspace)->profile.current_record
#define gc_prof_enabled(objspace) ((objspace)->profile.run && (objspace)->profile.current_record)
#define gc_report(level, objspace, ...) \
if (!RGENGC_DEBUG_ENABLED(level)) {} else gc_report_body(level, objspace, __VA_ARGS__)
PRINTF_ARGS(static void gc_report_body(int level, rb_objspace_t *objspace, const char *fmt, ...), 3, 4);
static void gc_finalize_deferred(void *dmy);
static void gc_tdata_unsafe_drain_objspaces(rb_objspace_t **objspaces, size_t n);
#if USE_TICK_T
/* the following code is only for internal tuning. */
/* Source code to use RDTSC is quoted and modified from
* https://www.mcs.anl.gov/~kazutomo/rdtsc.html
* written by Kazutomo Yoshii <kazutomo@mcs.anl.gov>
*/
#if defined(__GNUC__) && defined(__i386__)
typedef unsigned long long tick_t;
#define PRItick "llu"
static inline tick_t
tick(void)
{
unsigned long long int x;
__asm__ __volatile__ ("rdtsc" : "=A" (x));
return x;
}
#elif defined(__GNUC__) && defined(__x86_64__)
typedef unsigned long long tick_t;
#define PRItick "llu"
static __inline__ tick_t
tick(void)
{
unsigned long hi, lo;
__asm__ __volatile__ ("rdtsc" : "=a"(lo), "=d"(hi));
return ((unsigned long long)lo)|( ((unsigned long long)hi)<<32);
}
#elif defined(__powerpc64__) && (GCC_VERSION_SINCE(4,8,0) || defined(__clang__))
typedef unsigned long long tick_t;
#define PRItick "llu"
static __inline__ tick_t
tick(void)
{
unsigned long long val = __builtin_ppc_get_timebase();
return val;
}
#elif defined(__aarch64__) && defined(__GNUC__)
typedef unsigned long tick_t;
#define PRItick "lu"
static __inline__ tick_t
tick(void)
{
unsigned long val;
__asm__ __volatile__ ("mrs %0, cntvct_el0" : "=r" (val));
return val;
}
#elif defined(_WIN32) && defined(_MSC_VER)
#include <intrin.h>
typedef unsigned __int64 tick_t;
#define PRItick "llu"
static inline tick_t
tick(void)
{
return __rdtsc();
}
#else /* use clock */
typedef clock_t tick_t;
#define PRItick "llu"
static inline tick_t
tick(void)
{
return clock();
}
#endif /* TSC */
#else /* USE_TICK_T */
#define MEASURE_LINE(expr) expr
#endif /* USE_TICK_T */
static inline VALUE check_rvalue_consistency(rb_objspace_t *objspace, const VALUE obj);
#define RVALUE_MARKED_BITMAP(obj) MARKED_IN_BITMAP(GET_HEAP_MARK_BITS(obj), (obj))
#define RVALUE_WB_UNPROTECTED_BITMAP(obj) MARKED_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(obj), (obj))
#define RVALUE_MARKING_BITMAP(obj) MARKED_IN_BITMAP(GET_HEAP_MARKING_BITS(obj), (obj))
#define RVALUE_UNCOLLECTIBLE_BITMAP(obj) MARKED_IN_BITMAP(GET_HEAP_UNCOLLECTIBLE_BITS(obj), (obj))
#define RVALUE_PINNED_BITMAP(obj) MARKED_IN_BITMAP(GET_HEAP_PINNED_BITS(obj), (obj))
static inline int
RVALUE_MARKED(rb_objspace_t *objspace, VALUE obj)
{
check_rvalue_consistency(objspace, obj);
return RVALUE_MARKED_BITMAP(obj) != 0;
}
static inline int
RVALUE_PINNED(rb_objspace_t *objspace, VALUE obj)
{
check_rvalue_consistency(objspace, obj);
return RVALUE_PINNED_BITMAP(obj) != 0;
}
static inline int
RVALUE_WB_UNPROTECTED(rb_objspace_t *objspace, VALUE obj)
{
check_rvalue_consistency(objspace, obj);
return RVALUE_WB_UNPROTECTED_BITMAP(obj) != 0;
}
static inline int
RVALUE_MARKING(rb_objspace_t *objspace, VALUE obj)
{
check_rvalue_consistency(objspace, obj);
return RVALUE_MARKING_BITMAP(obj) != 0;
}
static inline int
RVALUE_REMEMBERED(rb_objspace_t *objspace, VALUE obj)
{
check_rvalue_consistency(objspace, obj);
return MARKED_IN_BITMAP(GET_HEAP_PAGE(obj)->remembered_bits, obj) != 0;
}
static inline int
RVALUE_UNCOLLECTIBLE(rb_objspace_t *objspace, VALUE obj)
{
check_rvalue_consistency(objspace, obj);
return RVALUE_UNCOLLECTIBLE_BITMAP(obj) != 0;
}
#define RVALUE_PAGE_WB_UNPROTECTED(page, obj) MARKED_IN_BITMAP((page)->wb_unprotected_bits, (obj))
#define RVALUE_PAGE_UNCOLLECTIBLE(page, obj) MARKED_IN_BITMAP((page)->uncollectible_bits, (obj))
#define RVALUE_PAGE_MARKING(page, obj) MARKED_IN_BITMAP((page)->marking_bits, (obj))
static void rgengc_remember(rb_objspace_t *objspace, VALUE obj);
static void gc_bitmaps_clear(rb_objspace_t *objspace, rb_heap_t *heap, bool clear_shref);
static void rgengc_rememberset_mark(rb_objspace_t *objspace, rb_heap_t *heap);
static bool verify_pointer_in_any_heap_p(const void *ptr); /* cross-objspace ownership test */
static int
check_rvalue_consistency_force(rb_objspace_t *objspace, const VALUE obj, int terminate)
{
int err = 0;
/* Under a global GC the barrier stops every Ractor, so the cross-objspace walk
* below is safe without the VM lock. Sweeping an ownerless zombie objspace also
* leaves GET_RACTOR() NULL, and taking the lock here would dereference it. */
const bool world_stopped = objspace->flags.during_global_gc;
/* The VM lock protects the cross-objspace walk while other Ractors run and
* reallocate their heaps. Not taken while this objspace is in GC: pages are stable
* then, the cross-objspace walk needs the world stopped anyway, and the Ractor lock
* may already be held (Ractor -> VM order inversion). A global GC holds the barrier
* and needs no lock. */
const bool take_vm_lock = !world_stopped && !during_gc;
unsigned int lev = 0;
if (take_vm_lock) lev = RB_GC_VM_LOCK_NO_BARRIER();
{
if (SPECIAL_CONST_P(obj)) {
fprintf(stderr, "check_rvalue_consistency: %p is a special const.\n", (void *)obj);
err++;
}
else if (!is_pointer_to_heap(objspace, (void *)obj)) {
/* obj may be a legitimate cross-objspace reference (a shareable object, an
* in-flight shref payload); it is a non-object only if no objspace's heap
* holds it. A foreign object's mark/age/remembered bits belong to its
* owner and reading them would race its local GC: skip per-object checks. */
if (!world_stopped) {
/* A mid-local-GC verify holds no barrier, so other Ractors reallocate
* heap_pages.sorted under verify_pointer_in_any_heap_p's page_index
* read. Accept foreign pointers here; the global GC's world-stopped
* verify does the full existence check. */
}
else if (!verify_pointer_in_any_heap_p((void *)obj)) {
struct heap_page *empty_page = objspace->empty_pages;
while (empty_page) {
if ((uintptr_t)empty_page->body <= (uintptr_t)obj &&
(uintptr_t)obj < (uintptr_t)empty_page->body + HEAP_PAGE_SIZE) {
GC_ASSERT(heap_page_in_global_empty_pages_pool(objspace, empty_page));
fprintf(stderr, "check_rvalue_consistency: %p is in an empty page (%p).\n",
(void *)obj, (void *)empty_page);
err++;
goto skip;
}
empty_page = empty_page->free_next;
}
fprintf(stderr, "check_rvalue_consistency: %p is not a Ruby object.\n", (void *)obj);
err++;
skip:
;
}
}
else {
const int wb_unprotected_bit = RVALUE_WB_UNPROTECTED_BITMAP(obj) != 0;
const int uncollectible_bit = RVALUE_UNCOLLECTIBLE_BITMAP(obj) != 0;
const int mark_bit = RVALUE_MARKED_BITMAP(obj) != 0;
const int marking_bit = RVALUE_MARKING_BITMAP(obj) != 0;
const int remembered_bit = MARKED_IN_BITMAP(GET_HEAP_PAGE(obj)->remembered_bits, obj) != 0;
const int age = RVALUE_AGE_GET((VALUE)obj);
if (heap_page_in_global_empty_pages_pool(objspace, GET_HEAP_PAGE(obj))) {
fprintf(stderr, "check_rvalue_consistency: %s is in tomb page.\n", rb_obj_info(obj));
err++;
}
if (BUILTIN_TYPE(obj) == T_NONE) {
fprintf(stderr, "check_rvalue_consistency: %s is T_NONE.\n", rb_obj_info(obj));
err++;
}
if (BUILTIN_TYPE(obj) == T_ZOMBIE) {
fprintf(stderr, "check_rvalue_consistency: %s is T_ZOMBIE.\n", rb_obj_info(obj));
err++;
}
/* Do not run the memsize probe once an inconsistency (a T_NONE, say) was
* found: an rb_bug inside the probe would lose the real diagnosis. */
if (err == 0 && BUILTIN_TYPE(obj) != T_DATA) {
rb_obj_memsize_of((VALUE)obj);
}
/* check generation
*
* OLD == age == 3 && old-bitmap && mark-bit (except incremental marking)
*/
if (age > 0 && wb_unprotected_bit) {
fprintf(stderr, "check_rvalue_consistency: %s is not WB protected, but age is %d > 0.\n", rb_obj_info(obj), age);
err++;
}
if (!is_marking(objspace) && uncollectible_bit && !mark_bit) {
fprintf(stderr, "check_rvalue_consistency: %s is uncollectible, but is not marked while !gc.\n", rb_obj_info(obj));
err++;
}
if (!is_full_marking(objspace)) {
if (uncollectible_bit && age != RVALUE_OLD_AGE && !wb_unprotected_bit) {
fprintf(stderr, "check_rvalue_consistency: %s is uncollectible, but not old (age: %d) and not WB unprotected.\n",
rb_obj_info(obj), age);
err++;
}
if (remembered_bit && age != RVALUE_OLD_AGE) {
fprintf(stderr, "check_rvalue_consistency: %s is remembered, but not old (age: %d).\n",
rb_obj_info(obj), age);
err++;
}
}
/*
* check coloring
*
* marking:false marking:true
* marked:false white *invalid*
* marked:true black grey
*/
if (is_incremental_marking(objspace) && marking_bit) {
if (!is_marking(objspace) && !mark_bit) {
fprintf(stderr, "check_rvalue_consistency: %s is marking, but not marked.\n", rb_obj_info(obj));
err++;
}
}
}
}
if (take_vm_lock) RB_GC_VM_UNLOCK_NO_BARRIER(lev);
if (err > 0 && terminate) {
rb_bug("check_rvalue_consistency_force: there is %d errors.", err);
}
return err;
}
#if RGENGC_CHECK_MODE == 0
static inline VALUE
check_rvalue_consistency(rb_objspace_t *objspace, const VALUE obj)
{
return obj;
}
#else
static VALUE
check_rvalue_consistency(rb_objspace_t *objspace, const VALUE obj)
{
check_rvalue_consistency_force(objspace, obj, TRUE);
return obj;
}
#endif
static inline bool
gc_object_moved_p(rb_objspace_t *objspace, VALUE obj)
{
bool ret;
asan_unpoisoning_object(obj) {
ret = BUILTIN_TYPE(obj) == T_MOVED;
}
return ret;
}
static inline int
RVALUE_OLD_P(rb_objspace_t *objspace, VALUE obj)
{
GC_ASSERT(!RB_SPECIAL_CONST_P(obj));
check_rvalue_consistency(objspace, obj);
// Because this will only ever be called on GC controlled objects,
// we can use the faster _RAW function here
return RB_OBJ_PROMOTED_RAW(obj);
}
static inline void
RVALUE_PAGE_OLD_UNCOLLECTIBLE_SET(rb_objspace_t *objspace, struct heap_page *page, VALUE obj)
{
MARK_IN_BITMAP(&page->uncollectible_bits[0], obj);
/* Count a promotion in the object's own objspace: a global GC ages every objspace's
* slots from the driver, and counting them there would skew the other objspaces'
* old_objects and with it their major GC frequency. */
page->objspace->rgengc.old_objects++;
#if RGENGC_PROFILE >= 2
objspace->profile.total_promoted_count++;
objspace->profile.promoted_types[BUILTIN_TYPE(obj)]++;
#endif
}
static inline void
RVALUE_OLD_UNCOLLECTIBLE_SET(rb_objspace_t *objspace, VALUE obj)
{
RB_DEBUG_COUNTER_INC(obj_promote);
RVALUE_PAGE_OLD_UNCOLLECTIBLE_SET(objspace, GET_HEAP_PAGE(obj), obj);
}
/* set age to age+1 */
static inline void
RVALUE_AGE_INC(rb_objspace_t *objspace, VALUE obj)
{
int age = RVALUE_AGE_GET((VALUE)obj);
if (RGENGC_CHECK_MODE && age == RVALUE_OLD_AGE) {
rb_bug("RVALUE_AGE_INC: can not increment age of OLD object %s.", rb_obj_info(obj));
}
age++;
RVALUE_AGE_SET(obj, age);
if (age == RVALUE_OLD_AGE) {
RVALUE_OLD_UNCOLLECTIBLE_SET(objspace, obj);
}
check_rvalue_consistency(objspace, obj);
}
static inline void
RVALUE_AGE_SET_CANDIDATE(rb_objspace_t *objspace, VALUE obj)
{
check_rvalue_consistency(objspace, obj);
GC_ASSERT(!RVALUE_OLD_P(objspace, obj));
RVALUE_AGE_SET(obj, RVALUE_OLD_AGE - 1);
check_rvalue_consistency(objspace, obj);
}
static inline void
RVALUE_AGE_RESET(VALUE obj)
{
RVALUE_AGE_SET(obj, 0);
}
static inline void
RVALUE_DEMOTE(rb_objspace_t *objspace, VALUE obj)
{
check_rvalue_consistency(objspace, obj);
GC_ASSERT(RVALUE_OLD_P(objspace, obj));
if (!is_incremental_marking(objspace) && RVALUE_REMEMBERED(objspace, obj)) {
struct heap_page *page = GET_HEAP_PAGE(obj);
_CLEAR_IN_BITMAP(page->remembered_bits, page, obj);
}
CLEAR_IN_BITMAP(GET_HEAP_UNCOLLECTIBLE_BITS(obj), obj);
RVALUE_AGE_RESET(obj);
if (RVALUE_MARKED(objspace, obj)) {
/* symmetric with RVALUE_PAGE_OLD_UNCOLLECTIBLE_SET */
GET_HEAP_PAGE(obj)->objspace->rgengc.old_objects--;
}
check_rvalue_consistency(objspace, obj);
}
static inline int
RVALUE_BLACK_P(rb_objspace_t *objspace, VALUE obj)
{
return RVALUE_MARKED(objspace, obj) && !RVALUE_MARKING(objspace, obj);
}
static inline int
RVALUE_WHITE_P(rb_objspace_t *objspace, VALUE obj)
{
return !RVALUE_MARKED(objspace, obj);
}
bool
rb_gc_impl_user_gc_disabled_set(void *objspace_ptr, bool disable)
{
rb_objspace_t *objspace = objspace_ptr;
const bool was = objspace->flags.user_gc_disabled;
objspace->flags.user_gc_disabled = disable;
return was;
}
bool
rb_gc_impl_user_gc_disabled_p(void *objspace_ptr)
{
rb_objspace_t *objspace = objspace_ptr;
return objspace->flags.user_gc_disabled;
}
bool
rb_gc_impl_gc_enabled_p(void *objspace_ptr)
{
rb_objspace_t *objspace = objspace_ptr;
return !dont_gc_val();
}
void
rb_gc_impl_gc_enable(void *objspace_ptr)
{
rb_objspace_t *objspace = objspace_ptr;
dont_gc_off();
}
void
rb_gc_impl_gc_disable(void *objspace_ptr, bool finish_current_gc)
{
rb_objspace_t *objspace = objspace_ptr;
if (finish_current_gc) {
gc_rest(objspace);
}
dont_gc_on();
}
/* Finish an incremental mark or lazy sweep in progress without changing the enabled
* state. gc.c uses it to settle the only objspace just before the process goes
* multi-objspace. */
void
rb_gc_impl_gc_rest(void *objspace_ptr)
{
gc_rest(objspace_ptr);
}
/*
--------------------------- ObjectSpace -----------------------------
*/
static inline void *
calloc1(size_t n)
{
return calloc(1, n);
}
void
rb_gc_impl_set_event_hook(void *objspace_ptr, const rb_event_flag_t event)
{
rb_objspace_t *objspace = objspace_ptr;
/* FREEOBJ is main-objspace only (rb_objspace_set_event_hook masks it elsewhere). */
GC_ASSERT(!(event & RUBY_INTERNAL_EVENT_FREEOBJ) ||
objspace == global_objspace->main_objspace);
objspace->hook_events = event & RUBY_INTERNAL_EVENT_OBJSPACE_MASK;
}
unsigned long long
rb_gc_impl_get_total_time(void *objspace_ptr)
{
rb_objspace_t *objspace = objspace_ptr;
unsigned long long marking_time = objspace->profile.marking_time_ns;
unsigned long long sweeping_time = objspace->profile.sweeping_time_ns;
return marking_time + sweeping_time;
}
void
rb_gc_impl_set_measure_total_time(void *objspace_ptr, VALUE flag)
{
rb_objspace_t *objspace = objspace_ptr;
objspace->flags.measure_gc = RTEST(flag) ? TRUE : FALSE;
}
bool
rb_gc_impl_get_measure_total_time(void *objspace_ptr)
{
rb_objspace_t *objspace = objspace_ptr;
return objspace->flags.measure_gc;
}
static void
gc_process_stat_capture(const rb_objspace_t *objspace,
struct gc_process_stat_snapshot *out)
{
out->minor_gc_count = objspace->profile.minor_gc_count;
out->major_gc_count = objspace->profile.major_gc_count;
out->global_gc_invocation_count = objspace->profile.global_gc_invocation_count;
out->marking_time_ns = objspace->profile.marking_time_ns;
out->sweeping_time_ns = objspace->profile.sweeping_time_ns;
}
static void
gc_process_stat_publish(rb_objspace_t *objspace)
{
struct gc_process_stat_snapshot snap;
GC_ASSERT(objspace->profile.count == objspace->profile.minor_gc_count +
objspace->profile.major_gc_count + objspace->profile.global_gc_count);
gc_process_stat_capture(objspace, &snap);
rb_native_mutex_lock(&objspace->process_stat.lock);
objspace->process_stat.published = snap;
rb_native_mutex_unlock(&objspace->process_stat.lock);
}
static void
gc_process_stat_add(struct gc_process_stat_total *dst,
const struct gc_process_stat_snapshot *src)
{
dst->minor_gc_count += src->minor_gc_count;
dst->major_gc_count += src->major_gc_count;
dst->global_gc_invocation_count += src->global_gc_invocation_count;
dst->marking_time_ns += src->marking_time_ns;
dst->sweeping_time_ns += src->sweeping_time_ns;
}
/* garbage objects will be collected soon. */
bool
rb_gc_impl_garbage_object_p(void *objspace_ptr, VALUE ptr)
{
rb_objspace_t *objspace = objspace_ptr;
/* A foreign object is a live leaf: reading its type or mark bit would race the
* owner's local GC, so outside a global GC's barrier never report it as garbage.
* The fstring/symbol weak-set lookups do reach across objspaces, but those objects
* are born shareable and only a stop-the-world global GC collects them, so "not
* garbage" is correct. */
if (gc_skip_foreign_object_p(objspace, ptr)) {
return false;
}
/* Asking whether a freed (T_NONE), moved (T_MOVED), or finalized (T_ZOMBIE)
* object is garbage gives an unreliable answer: the slot may since have been
* reused for an unrelated object. A reference to one of these is stale and a
* bug in the caller. */
asan_unpoisoning_object(ptr) {
GC_ASSERT(BUILTIN_TYPE(ptr) != T_NONE);
GC_ASSERT(BUILTIN_TYPE(ptr) != T_MOVED);
GC_ASSERT(BUILTIN_TYPE(ptr) != T_ZOMBIE);
}
return is_lazy_sweeping(objspace) && GET_HEAP_PAGE(ptr)->flags.before_sweep &&
!RVALUE_MARKED(objspace, ptr);
}
struct rb_gc_vm_context *
rb_gc_impl_get_vm_context(void *objspace_ptr)
{
rb_objspace_t *objspace = objspace_ptr;
return &objspace->vm_context;
}
static void free_stack_chunks(mark_stack_t *);
static void mark_stack_free_cache(mark_stack_t *);
static void heap_page_free(rb_objspace_t *objspace, struct heap_page *page);
static inline void
gc_check_obj_in_page(struct heap_page *page, VALUE obj)
{
if (RGENGC_CHECK_MODE &&
/* obj should belong to page */
!(page->start <= (uintptr_t)obj &&
(uintptr_t)obj < ((uintptr_t)page->start + (page->total_slots * page->slot_size)) &&
obj % sizeof(VALUE) == 0)) {
rb_bug("gc_check_obj_in_page: %p is not rvalue.", (void *)obj);
}
}
static inline void
heap_page_add_free_region(rb_objspace_t *objspace, struct heap_page *page, VALUE obj)
{
rb_asan_unpoison_object(obj, false);
// Should have already been reset
GC_ASSERT(RVALUE_AGE_GET(obj) == 0);
gc_check_obj_in_page(page, obj);
asan_unlock_freelist(page);
/* Keep a freed slot from carrying its old shareable and shref bits into the next
* object born there. */
CLEAR_IN_BITMAP(GET_HEAP_SHAREABLE_BITS(obj), obj);
CLEAR_IN_BITMAP(GET_HEAP_SHREF_BITS(obj), obj);
struct free_region *region = (struct free_region *)obj;
region->flags = 0;
region->end = (uintptr_t)obj + page->slot_size;
region->next = page->free_region;
page->free_region = region;
asan_lock_freelist(page);
rb_asan_poison_object(obj);
gc_report(3, objspace, "heap_page_add_free_region: %p\n", (void *)obj);
}
/* The initial size is per objspace, so a Ractor's own gets a smaller one than
* main's rather than paying main's again. */
static inline size_t
objspace_heap_init_bytes(const rb_objspace_t *objspace)
{
return objspace == global_objspace->main_objspace
? gc_params.heap_init_bytes : gc_params.ractor_heap_init_bytes;
}
static void
heap_allocatable_bytes_expand(rb_objspace_t *objspace,
rb_heap_t *heap, size_t free_slots, size_t total_slots, size_t slot_size)
{
double goal_ratio = gc_params.heap_free_slots_goal_ratio;
size_t target_total_slots;
if (goal_ratio == 0.0) {
target_total_slots = (size_t)(total_slots * gc_params.growth_factor);
}
else if (total_slots == 0) {
target_total_slots = objspace_heap_init_bytes(objspace) / slot_size;
}
else {
/* Find `f' where free_slots = f * total_slots * goal_ratio
* => f = (total_slots - free_slots) / ((1 - goal_ratio) * total_slots)
*/
double f = (double)(total_slots - free_slots) / ((1 - goal_ratio) * total_slots);
if (f > gc_params.growth_factor) f = gc_params.growth_factor;
if (f < 1.0) f = 1.1;
target_total_slots = (size_t)(f * total_slots);
if (0) {
fprintf(stderr,
"free_slots(%8"PRIuSIZE")/total_slots(%8"PRIuSIZE")=%1.2f,"
" G(%1.2f), f(%1.2f),"
" total_slots(%8"PRIuSIZE") => target_total_slots(%8"PRIuSIZE")\n",
free_slots, total_slots, free_slots/(double)total_slots,
goal_ratio, f, total_slots, target_total_slots);
}
}
if (gc_params.growth_max_bytes > 0) {
size_t max_total_slots = total_slots + gc_params.growth_max_bytes / slot_size;
if (target_total_slots > max_total_slots) target_total_slots = max_total_slots;
}
size_t extend_slot_count = target_total_slots - total_slots;
/* Extend by at least 1 page. */
if (extend_slot_count == 0) extend_slot_count = 1;
objspace->heap_pages.allocatable_bytes += extend_slot_count * slot_size;
}
static inline void
heap_add_freepage(rb_heap_t *heap, struct heap_page *page)
{
asan_unlock_freelist(page);
GC_ASSERT(page->free_slots != 0);
GC_ASSERT(page->free_region != NULL);
page->free_next = heap->free_pages;
heap->free_pages = page;
RUBY_DEBUG_LOG("page:%p free_region:%p", (void *)page, (void *)page->free_region);
asan_lock_freelist(page);
}
static inline void
heap_add_poolpage(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *page)
{
asan_unlock_freelist(page);
GC_ASSERT(page->free_slots != 0);
GC_ASSERT(page->free_region != NULL);
page->free_next = heap->pooled_pages;
heap->pooled_pages = page;
objspace->rincgc.pooled_slots += page->free_slots;
asan_lock_freelist(page);
}
static void
heap_unlink_page(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *page)
{
ccan_list_del(&page->page_node);
heap->total_pages--;
heap->total_slots -= page->total_slots;
}
/* Detach a page with no living objects from its heap and park it on the objspace's
* empty pages, from where it is reused or released to the page pool. The caller has
* already taken it off the heap's free_pages chain, if it was on it. */
static void
heap_page_detach_empty(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *page)
{
heap_unlink_page(objspace, heap, page);
page->start = 0;
page->total_slots = 0;
page->slot_size = 0;
page->heap = NULL;
page->free_slots = 0;
asan_unlock_freelist(page);
page->free_region = NULL;
asan_lock_freelist(page);
asan_poison_memory_region(page->body, HEAP_PAGE_SIZE);
objspace->empty_pages_count++;
page->free_next = objspace->empty_pages;
objspace->empty_pages = page;
}
static void
gc_aligned_free(void *ptr, size_t size)
{
#if defined __MINGW32__
__mingw_aligned_free(ptr);
#elif defined _WIN32
_aligned_free(ptr);
#elif defined(HAVE_POSIX_MEMALIGN) || defined(HAVE_MEMALIGN)
free(ptr);
#else
free(((void**)ptr)[-1]);
#endif
}
static void
heap_page_body_free(struct heap_page_body *page_body, struct page_arena *arena)
{
GC_ASSERT((uintptr_t)page_body % HEAP_PAGE_ALIGN == 0);
page_pool_release(page_body, arena);
}
#ifdef PAGE_POOL_LOCK_ERRORCHECK
# define ASSERT_PAGE_POOL_LOCKED(g) GC_ASSERT(pthread_mutex_lock(&(g)->page_pool.lock) == EDEADLK)
#else
# define ASSERT_PAGE_POOL_LOCKED(g) ((void)0)
#endif
/* Insert into page_index. Writers serialize on page_pool.lock; lomem and himem are a
* monotonically growing over-approximation used for a quick reject. */
static void
global_page_index_insert(struct heap_page *page)
{
rb_global_objspace_t *g = global_objspace;
uintptr_t body = (uintptr_t)page->body;
rb_native_mutex_lock(&g->page_pool.lock);
if (g->page_index.n_pages == g->page_index.capa) {
size_t new_capa = g->page_index.capa ? g->page_index.capa * 2 : 128;
struct heap_page **grown = realloc(g->page_index.pages, new_capa * sizeof(*grown));
if (grown == NULL) rb_bug("global_page_index_insert: realloc failed");
g->page_index.pages = grown;
g->page_index.capa = new_capa;
}
size_t lo = 0, hi = g->page_index.n_pages;
while (lo < hi) {
size_t mid = (lo + hi) / 2;
if ((uintptr_t)g->page_index.pages[mid]->body < body) lo = mid + 1;
else hi = mid;
}
memmove(&g->page_index.pages[lo + 1], &g->page_index.pages[lo],
(g->page_index.n_pages - lo) * sizeof(struct heap_page *));
g->page_index.pages[lo] = page;
g->page_index.n_pages++;
uintptr_t start = body + sizeof(struct heap_page_header);
uintptr_t end = body + HEAP_PAGE_SIZE;
if (g->page_index.lomem == 0 || g->page_index.lomem > start) g->page_index.lomem = start;
if (g->page_index.himem < end) g->page_index.himem = end;
rb_native_mutex_unlock(&g->page_pool.lock);
}
static void
global_page_index_remove_locked(const struct heap_page *page)
{
rb_global_objspace_t *g = global_objspace;
uintptr_t body = (uintptr_t)page->body;
ASSERT_PAGE_POOL_LOCKED(g);
size_t lo = 0, hi = g->page_index.n_pages;
while (lo < hi) {
size_t mid = (lo + hi) / 2;
if ((uintptr_t)g->page_index.pages[mid]->body < body) lo = mid + 1;
else hi = mid;
}
GC_ASSERT(lo < g->page_index.n_pages && g->page_index.pages[lo] == page);
memmove(&g->page_index.pages[lo], &g->page_index.pages[lo + 1],
(g->page_index.n_pages - lo - 1) * sizeof(struct heap_page *));
g->page_index.n_pages--;
}
static void
global_page_index_remove(const struct heap_page *page)
{
rb_global_objspace_t *g = global_objspace;
rb_native_mutex_lock(&g->page_pool.lock);
global_page_index_remove_locked(page);
rb_native_mutex_unlock(&g->page_pool.lock);
}
static void
heap_page_free(rb_objspace_t *objspace, struct heap_page *page)
{
global_page_index_remove(page);
objspace->heap_pages.freed_pages++;
heap_page_body_free(page->body, page->arena);
free(page);
}
static void
heap_pages_free_batch(rb_objspace_t *objspace, struct heap_page *pages)
{
rb_global_objspace_t *g = global_objspace;
rb_native_mutex_lock(&g->page_pool.lock);
for (struct heap_page *page = pages; page != NULL; page = page->free_next) {
global_page_index_remove_locked(page);
if (HEAP_PAGE_ALLOC_USE_MMAP) {
#ifdef HAVE_MMAP
page_pool_release_locked(page->body, page->arena);
#endif
}
}
rb_native_mutex_unlock(&g->page_pool.lock);
if (!HEAP_PAGE_ALLOC_USE_MMAP) {
/* gc_aligned_free does not need the pool lock. */
for (struct heap_page *page = pages; page != NULL; page = page->free_next) {
heap_page_body_free(page->body, page->arena);
}
}
while (pages != NULL) {
struct heap_page *next = pages->free_next;
objspace->heap_pages.freed_pages++;
free(pages);
pages = next;
}
}
static void
heap_pages_free_unused_pages(rb_objspace_t *objspace)
{
if (objspace->empty_pages != NULL && heap_pages_freeable_pages > 0) {
GC_ASSERT(objspace->empty_pages_count > 0);
objspace->empty_pages = NULL;
objspace->empty_pages_count = 0;
size_t i, j;
struct heap_page *to_free = NULL;
for (i = j = 0; i < rb_darray_size(objspace->heap_pages.sorted); i++) {
struct heap_page *page = rb_darray_get(objspace->heap_pages.sorted, i);
if (heap_page_in_global_empty_pages_pool(objspace, page) && heap_pages_freeable_pages > 0) {
page->free_next = to_free;
to_free = page;
heap_pages_freeable_pages--;
}
else {
if (heap_page_in_global_empty_pages_pool(objspace, page)) {
page->free_next = objspace->empty_pages;
objspace->empty_pages = page;
objspace->empty_pages_count++;
}
if (i != j) {
rb_darray_set(objspace->heap_pages.sorted, j, page);
}
j++;
}
}
rb_darray_pop(objspace->heap_pages.sorted, i - j);
GC_ASSERT(rb_darray_size(objspace->heap_pages.sorted) == j);
/* A retire GC can free every page, so an empty objspace is legitimate. */
if (j > 0) {
struct heap_page *hipage = rb_darray_get(objspace->heap_pages.sorted, rb_darray_size(objspace->heap_pages.sorted) - 1);
uintptr_t himem = (uintptr_t)hipage->body + HEAP_PAGE_SIZE;
GC_ASSERT(himem <= heap_pages_himem);
heap_pages_himem = himem;
struct heap_page *lopage = rb_darray_get(objspace->heap_pages.sorted, 0);
uintptr_t lomem = (uintptr_t)lopage->body + sizeof(struct heap_page_header);
GC_ASSERT(lomem >= heap_pages_lomem);
heap_pages_lomem = lomem;
}
else {
heap_pages_lomem = 0;
heap_pages_himem = 0;
}
heap_pages_free_batch(objspace, to_free);
}
}
static void *
gc_aligned_malloc(size_t alignment, size_t size)
{
/* alignment must be a power of 2 */
GC_ASSERT(((alignment - 1) & alignment) == 0);
GC_ASSERT(alignment % sizeof(void*) == 0);
void *res;
#if defined __MINGW32__
res = __mingw_aligned_malloc(size, alignment);
#elif defined _WIN32
res = _aligned_malloc(size, alignment);
#elif defined(HAVE_POSIX_MEMALIGN)
if (posix_memalign(&res, alignment, size) != 0) {
return NULL;
}
#elif defined(HAVE_MEMALIGN)
res = memalign(alignment, size);
#else
char* aligned;
res = malloc(alignment + size + sizeof(void*));
aligned = (char*)res + alignment + sizeof(void*);
aligned -= ((VALUE)aligned & (alignment - 1));
((void**)aligned)[-1] = res;
res = (void*)aligned;
#endif
GC_ASSERT((uintptr_t)res % alignment == 0);
return res;
}
/* The page pool (global_objspace->page_pool): heap page bodies are carved out of large
* arenas and reused through the pool. Free bodies are split into a small global hot
* list (≤ PAGE_POOL_HOT_MAX, never madvise'd) and per-arena cold freelists (eligible for
* OS release — see page_pool_reclaim). Both lists use an in-body link at offset 0. */
#define PAGE_POOL_ARENA_SIZE (HEAP_PAGE_SIZE * 32) /* 2MiB with 64KiB pages */
#define PAGE_POOL_ARENA_BODIES (PAGE_POOL_ARENA_SIZE / HEAP_PAGE_SIZE) /* 32 */
#define PAGE_POOL_HOT_MAX 0 /* disabled — empty_pages is the retention buffer */
#define PAGE_POOL_ARENA_KEEP_HALF (PAGE_POOL_ARENA_BODIES / 2) /* 16 */
/* Steal bit 0 of the in-body link word: set iff the body has been madvise'd (cold). */
#define PAGE_POOL_ADVISED_BIT ((uintptr_t)1)
/* While a body is free, the arena back-pointer is stored at offset sizeof(header) — one
* word past the link, inside the spared first OS page. PAGE_POOL_SCRATCH_SIZE covers
* both the link (offset 0) and the tag for ASAN unpoison. */
#define PAGE_POOL_BODY_ARENA(body) \
(*(struct page_arena **)((char *)(body) + sizeof(struct heap_page_header)))
#define PAGE_POOL_SCRATCH_SIZE (sizeof(struct heap_page_header) + sizeof(void *))
#ifdef HAVE_MMAP
/* mmap a new arena to carve from. Called with the pool lock held, at which point the
* previous arena is always fully carved. */
static bool
page_pool_add_arena(rb_global_objspace_t *g)
{
GC_ASSERT(HEAP_PAGE_ALIGN % sysconf(_SC_PAGE_SIZE) == 0);
size_t mmap_size = PAGE_POOL_ARENA_SIZE + HEAP_PAGE_ALIGN;
char *ptr = mmap(NULL, mmap_size,
PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
if (ptr == MAP_FAILED) {
return false;
}
// If we are building `default.c` as part of the ruby executable, we
// may just call `ruby_annotate_mmap`. But if we are building
// `default.c` as a shared library, we will not have access to private
// symbols, and we have to either call prctl directly or make our own
// wrapper.
#if defined(HAVE_SYS_PRCTL_H) && defined(PR_SET_VMA) && defined(PR_SET_VMA_ANON_NAME)
prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, ptr, mmap_size, "Ruby:GC:default:page_pool_arena");
errno = 0;
#endif
/* Trim the unaligned head and tail so the usable area is HEAP_PAGE_ALIGN aligned. */
char *aligned = ptr + HEAP_PAGE_ALIGN;
aligned -= ((uintptr_t)aligned & (HEAP_PAGE_ALIGN - 1));
GC_ASSERT(aligned > ptr);
GC_ASSERT(aligned <= ptr + HEAP_PAGE_ALIGN);
size_t start_out_of_range_size = aligned - ptr;
GC_ASSERT(start_out_of_range_size % sysconf(_SC_PAGE_SIZE) == 0);
if (start_out_of_range_size > 0) {
if (munmap(ptr, start_out_of_range_size)) {
rb_bug("page_pool_add_arena: munmap failed for start");
}
}
size_t end_out_of_range_size = HEAP_PAGE_ALIGN - start_out_of_range_size;
GC_ASSERT(end_out_of_range_size % sysconf(_SC_PAGE_SIZE) == 0);
if (end_out_of_range_size > 0) {
if (munmap(aligned + PAGE_POOL_ARENA_SIZE, end_out_of_range_size)) {
rb_bug("page_pool_add_arena: munmap failed for end");
}
}
struct page_arena *arena = calloc1(sizeof(struct page_arena));
if (arena == NULL) {
if (munmap(aligned, PAGE_POOL_ARENA_SIZE)) {
rb_bug("page_pool_add_arena: munmap failed for arena");
}
return false;
}
arena->start = aligned;
arena->size = PAGE_POOL_ARENA_SIZE;
arena->cold_freelist = NULL;
arena->free_count = 0;
arena->cold_count = 0;
arena->next = g->page_pool.arenas;
g->page_pool.arenas = arena;
g->page_pool.arena_count++;
g->page_pool.arena_current = arena;
g->page_pool.arena_cursor = aligned;
g->page_pool.arena_end = aligned + PAGE_POOL_ARENA_SIZE;
return true;
}
#endif
static struct heap_page_body *
page_pool_acquire(struct page_arena **arena_out)
{
struct heap_page_body *body = NULL;
if (HEAP_PAGE_ALLOC_USE_MMAP) {
#ifdef HAVE_MMAP
bool need_reuse = false;
rb_global_objspace_t *g = global_objspace;
rb_native_mutex_lock(&g->page_pool.lock);
if (g->page_pool.hot_list != NULL) {
body = g->page_pool.hot_list;
asan_unpoison_memory_region(body, PAGE_POOL_SCRATCH_SIZE, false);
uintptr_t link = *(uintptr_t *)body;
g->page_pool.hot_list = (struct heap_page_body *)(link & ~PAGE_POOL_ADVISED_BIT);
g->page_pool.hot_count--;
struct page_arena *arena = PAGE_POOL_BODY_ARENA(body);
arena->free_count--;
*arena_out = arena;
}
else {
// find cold page body (madvised reusable)
for (struct page_arena *a = g->page_pool.arenas; a; a = a->next) {
if (a->cold_count > 0) {
body = a->cold_freelist;
asan_unpoison_memory_region(body, PAGE_POOL_SCRATCH_SIZE, false);
uintptr_t link = *(uintptr_t *)body;
a->cold_freelist = (struct heap_page_body *)(link & ~PAGE_POOL_ADVISED_BIT);
a->cold_count--;
a->free_count--;
*arena_out = a;
need_reuse = (link & PAGE_POOL_ADVISED_BIT) != 0;
if (need_reuse) g->page_pool.advised_count--;
break;
}
}
if (body == NULL &&
(g->page_pool.arena_cursor != g->page_pool.arena_end ||
page_pool_add_arena(g))) {
GC_ASSERT(g->page_pool.arena_cursor + HEAP_PAGE_SIZE <= g->page_pool.arena_end);
body = (struct heap_page_body *)g->page_pool.arena_cursor;
g->page_pool.arena_cursor += HEAP_PAGE_SIZE;
*arena_out = g->page_pool.arena_current;
}
}
rb_native_mutex_unlock(&g->page_pool.lock);
if (body != NULL) {
if (need_reuse) {
rb_vm_map_reuse((char *)body + g->page_pool.os_page_size,
HEAP_PAGE_SIZE - g->page_pool.os_page_size);
}
asan_unpoison_memory_region(body, HEAP_PAGE_SIZE, false);
}
#endif
}
else {
body = gc_aligned_malloc(HEAP_PAGE_ALIGN, HEAP_PAGE_SIZE);
*arena_out = NULL;
}
return body;
}
#ifdef HAVE_MMAP
static void
page_pool_release_locked(struct heap_page_body *body, struct page_arena *arena)
{
rb_global_objspace_t *g = global_objspace;
ASSERT_PAGE_POOL_LOCKED(g);
/* A body in the empty-pages pool stays fully poisoned (see gc_sweep_page), so
* unpoison the scratch area (link + arena tag) before writing. */
asan_unpoison_memory_region(body, PAGE_POOL_SCRATCH_SIZE, false);
arena->free_count++;
PAGE_POOL_BODY_ARENA(body) = arena;
if (g->page_pool.hot_count < PAGE_POOL_HOT_MAX) {
*(uintptr_t *)body = (uintptr_t)g->page_pool.hot_list;
g->page_pool.hot_list = body;
g->page_pool.hot_count++;
}
else {
*(uintptr_t *)body = (uintptr_t)arena->cold_freelist;
arena->cold_freelist = body;
arena->cold_count++;
}
asan_poison_memory_region(body, HEAP_PAGE_SIZE);
}
#endif
static void
page_pool_release(struct heap_page_body *body, struct page_arena *arena)
{
if (HEAP_PAGE_ALLOC_USE_MMAP) {
#ifdef HAVE_MMAP
rb_global_objspace_t *g = global_objspace;
rb_native_mutex_lock(&g->page_pool.lock);
page_pool_release_locked(body, arena);
rb_native_mutex_unlock(&g->page_pool.lock);
#endif
}
else {
gc_aligned_free(body, HEAP_PAGE_SIZE);
}
}
/* Allow the OS to reclaim pool memory. Runs only at major GC in single-objspace mode
* (see gc_sweep_finish).
*
* Step A: madvise cold bodies, sparing the first OS page (which holds the in-body
* freelist link and arena tag).
*
* Step B: munmap arenas whose 32 bodies are all free, keeping one extra empty
* arena as a retention buffer when the remaining free pool is < half an arena. */
static void
page_pool_reclaim(rb_global_objspace_t *g)
{
if (!HEAP_PAGE_ALLOC_USE_MMAP) return;
#ifdef HAVE_MMAP
size_t os_page_size = g->page_pool.os_page_size;
rb_native_mutex_lock(&g->page_pool.lock);
/* Advising spares the first OS page of a body (it holds the in-body freelist link
* and the arena tag), so it needs sub-page granularity: when the OS page size is
* >= HEAP_PAGE_SIZE (e.g. 64KiB pages on aarch64) no body is ever advised, and
* advised_count must not be adjusted anywhere either. */
const bool can_advise = os_page_size < HEAP_PAGE_SIZE;
/* Step A — advise cold bodies (immediate release: drop RSS now if the platform allows). */
if (can_advise) {
for (struct page_arena *a = g->page_pool.arenas; a; a = a->next) {
for (struct heap_page_body *body = a->cold_freelist; body; ) {
asan_unpoison_memory_region(body, PAGE_POOL_SCRATCH_SIZE, false);
uintptr_t link = *(uintptr_t *)body;
struct heap_page_body *next =
(struct heap_page_body *)(link & ~PAGE_POOL_ADVISED_BIT);
if (!(link & PAGE_POOL_ADVISED_BIT)) {
rb_vm_map_reusable_immediate((char *)body + os_page_size,
HEAP_PAGE_SIZE - os_page_size, 0);
*(uintptr_t *)body = link | PAGE_POOL_ADVISED_BIT;
g->page_pool.advised_count++;
}
asan_poison_memory_region(body, PAGE_POOL_SCRATCH_SIZE);
body = next;
}
}
}
/* Step B — munmap fully-free arenas (with retention buffer).
*
* total_free = Σ free_count; free_count already includes hot-list bodies
* (page_pool_release increments it unconditionally), so no separate hot_count.
* An arena is eligible when all 32 of its bodies are free AND none sit on
* the hot list (≤5 entries, pre-scanned). Keep one extra empty arena when
* the rest of the free pool is < half an arena, to avoid thrash. */
int total_free = 0;
for (struct page_arena *a = g->page_pool.arenas; a; a = a->next) {
total_free += a->free_count;
}
struct page_arena *hot_arenas[PAGE_POOL_HOT_MAX ? PAGE_POOL_HOT_MAX : 1];
int n_hot_arenas = 0;
/* Collect arenas that have a hot body (≤ PAGE_POOL_HOT_MAX entries). */
for (struct heap_page_body *body = g->page_pool.hot_list; body; ) {
asan_unpoison_memory_region(body, PAGE_POOL_SCRATCH_SIZE, false);
uintptr_t link = *(uintptr_t *)body;
struct heap_page_body *next =
(struct heap_page_body *)(link & ~PAGE_POOL_ADVISED_BIT);
struct page_arena *arena = PAGE_POOL_BODY_ARENA(body);
bool found = false;
for (int i = 0; i < n_hot_arenas; i++) {
if (hot_arenas[i] == arena) { found = true; break; }
}
if (!found && n_hot_arenas < PAGE_POOL_HOT_MAX) {
hot_arenas[n_hot_arenas++] = arena;
}
asan_poison_memory_region(body, PAGE_POOL_SCRATCH_SIZE);
body = next;
}
bool retained_one = false;
struct page_arena **pp = &g->page_pool.arenas;
// munmap fully free arenas
while (*pp) {
struct page_arena *a = *pp;
bool has_hot = false;
for (int i = 0; i < n_hot_arenas; i++) {
if (hot_arenas[i] == a) { has_hot = true; break; }
}
if (a->free_count != PAGE_POOL_ARENA_BODIES || has_hot) {
pp = &a->next;
continue;
}
GC_ASSERT(a->cold_count == PAGE_POOL_ARENA_BODIES);
int free_elsewhere = total_free - PAGE_POOL_ARENA_BODIES;
if (free_elsewhere < PAGE_POOL_ARENA_KEEP_HALF && !retained_one) {
retained_one = true;
pp = &a->next;
continue;
}
*pp = a->next;
if (munmap(a->start, a->size)) {
rb_bug("page_pool_reclaim: munmap failed");
}
total_free -= PAGE_POOL_ARENA_BODIES;
/* Every body of this arena is on its cold freelist, so Step A above has just
* advised all of them -- but only if this platform can advise at all. */
if (can_advise) {
g->page_pool.advised_count -= PAGE_POOL_ARENA_BODIES;
GC_ASSERT(g->page_pool.advised_count >= 0);
}
g->page_pool.arena_count--;
g->page_pool.arenas_unmapped++;
if (a == g->page_pool.arena_current) {
// During next acquire, any remaining arenas that have cold bodies are used. This is guaranteed
// because of the retention buffer.
g->page_pool.arena_current = NULL;
g->page_pool.arena_cursor = NULL;
g->page_pool.arena_end = NULL;
}
free(a);
}
rb_native_mutex_unlock(&g->page_pool.lock);
#endif
}
static struct heap_page_body *
heap_page_body_allocate(struct page_arena **arena_out)
{
struct heap_page_body *page_body = page_pool_acquire(arena_out);
GC_ASSERT(page_body == NULL || (uintptr_t)page_body % HEAP_PAGE_ALIGN == 0);
return page_body;
}
static struct heap_page *
heap_page_resurrect(rb_objspace_t *objspace)
{
struct heap_page *page = NULL;
if (objspace->empty_pages == NULL) {
GC_ASSERT(objspace->empty_pages_count == 0);
}
else {
GC_ASSERT(objspace->empty_pages_count > 0);
objspace->empty_pages_count--;
page = objspace->empty_pages;
objspace->empty_pages = page->free_next;
/* Clear the flags left over from emptying the page before reusing it, or the
* shareable and shref scans would keep walking an empty bitmap forever. */
page->flags.has_shareable_objects = FALSE;
page->flags.has_shref_objects = FALSE;
}
return page;
}
static struct heap_page *
heap_page_allocate(rb_objspace_t *objspace)
{
struct page_arena *arena;
struct heap_page_body *page_body = heap_page_body_allocate(&arena);
if (page_body == 0) {
rb_memerror();
}
struct heap_page *page = calloc1(sizeof(struct heap_page));
if (page == 0) {
heap_page_body_free(page_body, arena);
rb_memerror();
}
uintptr_t start = (uintptr_t)page_body + sizeof(struct heap_page_header);
uintptr_t end = (uintptr_t)page_body + HEAP_PAGE_SIZE;
size_t lo = 0;
size_t hi = rb_darray_size(objspace->heap_pages.sorted);
while (lo < hi) {
struct heap_page *mid_page;
size_t mid = (lo + hi) / 2;
mid_page = rb_darray_get(objspace->heap_pages.sorted, mid);
if ((uintptr_t)mid_page->start < start) {
lo = mid + 1;
}
else if ((uintptr_t)mid_page->start > start) {
hi = mid;
}
else {
rb_bug("same heap page is allocated: %p at %"PRIuVALUE, (void *)page_body, (VALUE)mid);
}
}
rb_darray_insert_without_gc(&objspace->heap_pages.sorted, hi, page);
if (heap_pages_lomem == 0 || heap_pages_lomem > start) heap_pages_lomem = start;
if (heap_pages_himem < end) heap_pages_himem = end;
page->body = page_body;
page->arena = arena;
page_body->header.page = page;
page->objspace = objspace;
objspace->heap_pages.allocated_pages++;
global_page_index_insert(page);
return page;
}
static void
heap_add_page(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *page)
{
/* Adding to eden heap during incremental sweeping is forbidden */
GC_ASSERT(!heap->sweeping_page);
GC_ASSERT(heap_page_in_global_empty_pages_pool(objspace, page));
/* Align start to slot_size boundary */
uintptr_t start = (uintptr_t)page->body + sizeof(struct heap_page_header);
uintptr_t rem = start % heap->slot_size;
if (rem) start += heap->slot_size - rem;
int slot_count = (int)((HEAP_PAGE_SIZE - (start - (uintptr_t)page->body))/heap->slot_size);
page->start = start;
page->total_slots = slot_count;
page->slot_size = heap->slot_size;
page->slot_size_reciprocal = heap_slot_reciprocal_table[heap - heaps];
page->heap = heap;
memset(&page->wb_unprotected_bits[0], 0, HEAP_PAGE_BITMAP_SIZE);
memset(&page->age_bits[0], 0, sizeof(page->age_bits));
asan_unlock_freelist(page);
asan_unpoison_memory_region(page->body, HEAP_PAGE_SIZE, false);
uintptr_t slots_end = start + (uintptr_t)slot_count * heap->slot_size;
memset((void *)start, 0, slots_end - start);
struct free_region *region = (struct free_region *)start;
region->flags = 0;
region->end = slots_end;
region->next = NULL;
page->free_region = region;
/* Poison every free slot; each is unpoisoned again as it is handed out. */
for (uintptr_t p = start; p < slots_end; p += heap->slot_size) {
rb_asan_poison_object((VALUE)p);
}
asan_lock_freelist(page);
page->free_slots = slot_count;
heap->total_allocated_pages++;
ccan_list_add_tail(&heap->pages, &page->page_node);
heap->total_pages++;
heap->total_slots += page->total_slots;
}
static int
heap_page_allocate_and_initialize(rb_objspace_t *objspace, rb_heap_t *heap)
{
gc_report(1, objspace, "heap_page_allocate_and_initialize: rb_darray_size(objspace->heap_pages.sorted): %"PRIdSIZE", "
"allocatable_bytes: %"PRIdSIZE", heap->total_pages: %"PRIdSIZE"\n",
rb_darray_size(objspace->heap_pages.sorted), objspace->heap_pages.allocatable_bytes, heap->total_pages);
bool allocated = false;
struct heap_page *page = heap_page_resurrect(objspace);
if (page == NULL && objspace->heap_pages.allocatable_bytes > 0) {
page = heap_page_allocate(objspace);
allocated = true;
GC_ASSERT(page != NULL);
}
if (page != NULL) {
heap_add_page(objspace, heap, page);
heap_add_freepage(heap, page);
if (allocated) {
size_t page_bytes = (size_t)page->total_slots * page->slot_size;
if (objspace->heap_pages.allocatable_bytes > page_bytes) {
objspace->heap_pages.allocatable_bytes -= page_bytes;
}
else {
objspace->heap_pages.allocatable_bytes = 0;
}
}
}
return page != NULL;
}
static void
heap_page_allocate_and_initialize_force(rb_objspace_t *objspace, rb_heap_t *heap)
{
size_t prev_allocatable_bytes = objspace->heap_pages.allocatable_bytes;
objspace->heap_pages.allocatable_bytes = HEAP_PAGE_SIZE;
heap_page_allocate_and_initialize(objspace, heap);
GC_ASSERT(heap->free_pages != NULL);
objspace->heap_pages.allocatable_bytes = prev_allocatable_bytes;
}
static void
gc_continue(rb_objspace_t *objspace, rb_heap_t *heap)
{
unsigned int lock_lev;
bool needs_gc = is_incremental_marking(objspace) || needs_continue_sweeping(objspace, heap);
if (!needs_gc) return;
gc_enter(objspace, gc_enter_event_continue, &lock_lev); // takes vm barrier, try to avoid
/* Continue marking if in incremental marking. */
if (is_incremental_marking(objspace)) {
if (gc_marks_continue(objspace, heap)) {
gc_sweep(objspace);
}
}
if (needs_continue_sweeping(objspace, heap)) {
gc_sweep_continue(objspace, heap);
}
gc_exit(objspace, gc_enter_event_continue, &lock_lev);
}
static void
heap_prepare(rb_objspace_t *objspace, rb_heap_t *heap)
{
GC_ASSERT(heap->free_pages == NULL);
if (heap->total_slots < objspace_heap_init_bytes(objspace) / heap->slot_size &&
heap->sweeping_page == NULL) {
heap_page_allocate_and_initialize_force(objspace, heap);
GC_ASSERT(heap->free_pages != NULL);
return;
}
/* Continue incremental marking or lazy sweeping, if in any of those steps. */
gc_continue(objspace, heap);
if (heap->free_pages == NULL) {
heap_page_allocate_and_initialize(objspace, heap);
}
/* If we still don't have a free page and not allowed to create a new page,
* we should start a new GC cycle. */
if (heap->free_pages == NULL) {
GC_ASSERT(objspace->empty_pages_count == 0);
GC_ASSERT(objspace->heap_pages.allocatable_bytes == 0);
if (gc_start(objspace, GPR_FLAG_NEWOBJ) == FALSE) {
rb_memerror();
}
else {
if (objspace->heap_pages.allocatable_bytes == 0 && !gc_config_full_mark_val) {
heap_allocatable_bytes_expand(objspace, heap,
heap->freed_slots + heap->empty_slots,
heap->total_slots, heap->slot_size);
GC_ASSERT(objspace->heap_pages.allocatable_bytes > 0);
}
/* Do steps of incremental marking or lazy sweeping if the GC run permits. */
gc_continue(objspace, heap);
/* If we're not incremental marking (e.g. a minor GC) or finished
* sweeping and still don't have a free page, then
* gc_sweep_finish_heap should allow us to create a new page. */
if (heap->free_pages == NULL && !heap_page_allocate_and_initialize(objspace, heap)) {
if (gc_needs_major_flags == GPR_FLAG_NONE) {
rb_bug("cannot create a new page after GC");
}
else { // Major GC is required, which will allow us to create new page
if (gc_start(objspace, GPR_FLAG_NEWOBJ) == FALSE) {
rb_memerror();
}
else {
/* Do steps of incremental marking or lazy sweeping. */
gc_continue(objspace, heap);
if (heap->free_pages == NULL &&
!heap_page_allocate_and_initialize(objspace, heap)) {
rb_bug("cannot create a new page after major GC");
}
}
}
}
}
}
GC_ASSERT(heap->free_pages != NULL);
}
#if GC_DEBUG
static inline const char*
rb_gc_impl_source_location_cstr(int *ptr)
{
/* We could directly refer `rb_source_location_cstr()` before, but not any
* longer. We have to heavy lift using our debugging API. */
if (! ptr) {
return NULL;
}
else if (! (*ptr = rb_sourceline())) {
return NULL;
}
else {
return rb_sourcefile();
}
}
#endif
static inline VALUE
newobj_init(VALUE klass, VALUE flags, int wb_protected, rb_objspace_t *objspace, VALUE obj)
{
GC_ASSERT(BUILTIN_TYPE(obj) == T_NONE);
GC_ASSERT((flags & FL_WB_PROTECTED) == 0);
RBASIC(obj)->flags = flags;
*((VALUE *)&RBASIC(obj)->klass) = klass;
#if RBASIC_SHAPE_ID_FIELD
RBASIC(obj)->shape_id = 0;
#endif
if (RB_UNLIKELY(flags & RUBY_FL_SHAREABLE)) {
/* A born-shareable object must be WB protected: the shref and remembered-set
* rules for shareable objects assume the write barrier. A local GC roots
* shareable objects from this bit (pinned_roots_mark). */
GC_ASSERT(wb_protected);
gc_page_add_shareable(GET_HEAP_PAGE(obj), obj);
}
#if RGENGC_CHECK_MODE
int lev = RB_GC_VM_LOCK_NO_BARRIER();
{
check_rvalue_consistency(objspace, obj);
GC_ASSERT(RVALUE_MARKED(objspace, obj) == FALSE);
GC_ASSERT(RVALUE_MARKING(objspace, obj) == FALSE);
GC_ASSERT(RVALUE_OLD_P(objspace, obj) == FALSE);
GC_ASSERT(RVALUE_WB_UNPROTECTED(objspace, obj) == FALSE);
if (RVALUE_REMEMBERED(objspace, obj)) rb_bug("newobj: %s is remembered.", rb_obj_info(obj));
}
RB_GC_VM_UNLOCK_NO_BARRIER(lev);
#endif
if (RB_UNLIKELY(wb_protected == FALSE)) {
MARK_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(obj), obj);
}
#if RGENGC_PROFILE
if (wb_protected) {
objspace->profile.total_generated_normal_object_count++;
#if RGENGC_PROFILE >= 2
objspace->profile.generated_normal_object_count_types[BUILTIN_TYPE(obj)]++;
#endif
}
else {
objspace->profile.total_generated_shady_object_count++;
#if RGENGC_PROFILE >= 2
objspace->profile.generated_shady_object_count_types[BUILTIN_TYPE(obj)]++;
#endif
}
#endif
#if GC_DEBUG
GET_RVALUE_OVERHEAD(obj)->file = rb_gc_impl_source_location_cstr(&GET_RVALUE_OVERHEAD(obj)->line);
GC_ASSERT(!SPECIAL_CONST_P(obj)); /* check alignment */
#endif
gc_report(5, objspace, "newobj: %s\n", rb_obj_info(obj));
// RUBY_DEBUG_LOG("obj:%p (%s)", (void *)obj, rb_obj_info(obj));
return obj;
}
size_t
rb_gc_impl_obj_slot_size(VALUE obj)
{
return GET_HEAP_PAGE(obj)->slot_size - RVALUE_OVERHEAD;
}
bool
rb_gc_impl_pinned_p(void *objspace_ptr, VALUE obj)
{
return RVALUE_PINNED((rb_objspace_t *)objspace_ptr, obj);
}
static inline size_t
heap_slot_size(unsigned char pool_id)
{
GC_ASSERT(pool_id < HEAP_COUNT);
return pool_slot_sizes[pool_id] - RVALUE_OVERHEAD;
}
size_t
rb_gc_impl_max_allocation_size(void)
{
return heap_slot_size(HEAP_COUNT - 1);
}
bool
rb_gc_impl_size_allocatable_p(size_t size)
{
return size <= rb_gc_impl_max_allocation_size();
}
static inline bool
heap_advance_region(rb_heap_t *heap)
{
struct free_region *region = heap->newobj.alloc_next_region;
if (region == NULL) {
return false;
}
rb_asan_unpoison_object((VALUE)region, false);
GC_ASSERT(RB_TYPE_P((VALUE)region, T_NONE));
heap->newobj.alloc_cursor = (uintptr_t)region;
heap->newobj.alloc_cursor_end = region->end;
heap->newobj.alloc_next_region = region->next;
rb_asan_poison_object((VALUE)region);
return true;
}
/* The whole region is ours until the next refill, so charge it to the step now. */
static inline void
heap_charge_region(rb_objspace_t *objspace, const rb_heap_t *heap, size_t heap_idx)
{
objspace->incremental_mark_step_allocated_slots +=
(heap->newobj.alloc_cursor_end - heap->newobj.alloc_cursor) / pool_slot_sizes[heap_idx];
}
static inline VALUE
heap_alloc_slot(rb_objspace_t *objspace, size_t heap_idx)
{
rb_heap_t *heap = &heaps[heap_idx];
uintptr_t cursor = heap->newobj.alloc_cursor;
if (RB_UNLIKELY(cursor >= heap->newobj.alloc_cursor_end)) {
/* Marking owes us a step before the next region, and newobj_refill runs it. */
if (RB_UNLIKELY(is_incremental_marking(objspace)) ||
heap_advance_region(heap) == false) {
return Qfalse;
}
cursor = heap->newobj.alloc_cursor;
}
VALUE obj = (VALUE)cursor;
rb_asan_unpoison_object(obj, true);
heap->newobj.alloc_cursor = cursor + pool_slot_sizes[heap_idx];
/* Single writer (the owning Ractor under the GVL), so a plain increment is enough. */
heap->total_allocated_objects++;
#if RGENGC_CHECK_MODE
GC_ASSERT(rb_gc_impl_obj_slot_size(obj) == heap_slot_size(heap_idx));
// zero clear
MEMZERO((char *)obj, char, heap_slot_size(heap_idx));
#endif
return obj;
}
static struct heap_page *
heap_next_free_page(rb_objspace_t *objspace, rb_heap_t *heap)
{
struct heap_page *page;
if (heap->free_pages == NULL) {
heap_prepare(objspace, heap);
}
page = heap->free_pages;
heap->free_pages = page->free_next;
GC_ASSERT(page->free_slots != 0);
asan_unlock_freelist(page);
return page;
}
static inline void
heap_set_alloc_page(rb_objspace_t *objspace, size_t heap_idx, struct heap_page *page)
{
gc_report(3, objspace, "heap_set_alloc_page: Using page %p\n", (void *)page->body);
rb_heap_t *heap = &heaps[heap_idx];
GC_ASSERT(heap->newobj.alloc_cursor >= heap->newobj.alloc_cursor_end);
GC_ASSERT(heap->newobj.alloc_next_region == NULL);
GC_ASSERT(page->free_slots != 0);
GC_ASSERT(page->free_region != NULL);
heap->newobj.alloc_using_page = page;
struct free_region *region = page->free_region;
rb_asan_unpoison_object((VALUE)region, false);
GC_ASSERT(RB_TYPE_P((VALUE)region, T_NONE));
heap->newobj.alloc_cursor = (uintptr_t)region;
heap->newobj.alloc_cursor_end = region->end;
heap->newobj.alloc_next_region = region->next;
rb_asan_poison_object((VALUE)region);
page->free_slots = 0;
page->free_region = NULL;
}
static void
init_size_to_heap_idx(void)
{
/* Process-wide and immutable, so build it once at boot. A rebuild in a later
* objspace_init would write the same values but race other threads' lock-free
* allocation-fastpath reads. */
static bool initialized = false;
if (initialized) return;
initialized = true;
for (size_t i = 0; i < sizeof(size_to_heap_idx); i++) {
size_t effective = i * 8 + RVALUE_OVERHEAD;
uint8_t idx;
for (idx = 0; idx < HEAP_COUNT; idx++) {
if (effective <= pool_slot_sizes[idx]) break;
}
size_to_heap_idx[i] = idx;
}
}
static inline size_t
heap_idx_for_size(size_t size)
{
size_t compressed = (size + 7) >> 3;
if (compressed < sizeof(size_to_heap_idx)) {
size_t heap_idx = size_to_heap_idx[compressed];
if (RB_LIKELY(heap_idx < HEAP_COUNT)) return heap_idx;
}
rb_bug("heap_idx_for_size: allocation size too large "
"(size=%"PRIuSIZE")", size);
}
size_t
rb_gc_impl_size_slot_size(void *objspace_ptr, size_t size)
{
return heap_slot_size((unsigned char)heap_idx_for_size(size));
}
bool
rb_gc_impl_zjit_new_obj_fastpath(void *objspace_ptr, size_t alloc_size, VALUE flags, VALUE klass,
struct rb_gc_zjit_fastpath *fastpath)
{
#if USE_ZJIT
size_t heap_idx = 0;
size_t slot_size = 0;
for (; heap_idx < HEAP_COUNT; heap_idx++) {
if (alloc_size + RVALUE_OVERHEAD <= pool_slot_sizes[heap_idx]) {
slot_size = pool_slot_sizes[heap_idx];
break;
}
}
if (slot_size == 0) return false;
#undef heaps
size_t base = offsetof(rb_objspace_t, heaps)
+ heap_idx * sizeof(rb_heap_t)
+ offsetof(rb_heap_t, newobj);
#define heaps objspace->heaps
struct rb_gc_zjit_default_new_obj_fastpath default_fastpath = {
base + offsetof(rb_heap_newobj_t, alloc_cursor),
base + offsetof(rb_heap_newobj_t, alloc_cursor_end),
slot_size,
base - offsetof(rb_heap_t, newobj) + offsetof(rb_heap_t, total_allocated_objects),
flags,
klass
};
memset(fastpath, 0, sizeof(*fastpath));
fastpath->kind = RB_GC_ZJIT_FASTPATH_DEFAULT;
memcpy(fastpath->data.words, &default_fastpath, sizeof(default_fastpath));
return true;
#else
return false;
#endif
}
int32_t
rb_gc_impl_zjit_writebarrier_required_offset(void)
{
return (int32_t)offsetof(rb_objspace_t, zjit_during_incremental_marking);
}
NOINLINE(static VALUE newobj_refill(rb_objspace_t *objspace, size_t heap_idx));
static VALUE
newobj_refill(rb_objspace_t *objspace, size_t heap_idx)
{
rb_heap_t *heap = &heaps[heap_idx];
VALUE obj = Qfalse;
/* No lock: a heap is single-writer (its owner thread, serialized by the GVL inside
* the Ractor), the page pool has its own mutex, and a GC started from here takes
* whatever gc_enter needs. */
if (is_incremental_marking(objspace)) {
/* The fast path sends us here at every region, which is far more often than the
* step size, so step only once the regions add up to it. */
if (objspace->incremental_mark_step_allocated_slots >= INCREMENTAL_MARK_STEP_ALLOCATIONS) {
gc_continue(objspace, heap);
objspace->incremental_mark_step_allocated_slots = 0;
}
// Move on to the region the fast path refused to take
if (heap_advance_region(heap)) {
heap_charge_region(objspace, heap, heap_idx);
obj = heap_alloc_slot(objspace, heap_idx);
}
}
if (obj == Qfalse) {
// Get next free page (possibly running GC)
struct heap_page *page = heap_next_free_page(objspace, heap);
heap_set_alloc_page(objspace, heap_idx, page);
heap_charge_region(objspace, heap, heap_idx);
// Retry allocation after moving to new page
obj = heap_alloc_slot(objspace, heap_idx);
}
if (RB_UNLIKELY(obj == Qfalse)) {
rb_memerror();
}
return obj;
}
static VALUE
newobj_alloc(rb_objspace_t *objspace, size_t heap_idx)
{
/* The objspace belongs to the current Ractor and is single-writer, so the fast path
* needs no lock. Stress GC runs in the caller's slow path, before newobj_alloc. */
VALUE obj = heap_alloc_slot(objspace, heap_idx);
if (RB_UNLIKELY(obj == Qfalse)) {
obj = newobj_refill(objspace, heap_idx);
}
return obj;
}
ALWAYS_INLINE(static VALUE newobj_slowpath(VALUE klass, VALUE flags, rb_objspace_t *objspace, int wb_protected, size_t heap_idx));
static inline VALUE
newobj_slowpath(VALUE klass, VALUE flags, rb_objspace_t *objspace, int wb_protected, size_t heap_idx)
{
VALUE obj;
/* No lock (see newobj_refill); during_gc and the stress flag are this objspace's own state. */
if (RB_UNLIKELY(during_gc || ruby_gc_stressful)) {
if (during_gc) {
dont_gc_on();
during_gc = 0;
if (rb_memerror_reentered()) {
rb_memerror();
}
rb_bug("object allocation during garbage collection phase");
}
if (ruby_gc_stressful) {
if (!garbage_collect(objspace, GPR_FLAG_NEWOBJ)) {
rb_memerror();
}
}
}
obj = newobj_alloc(objspace, heap_idx);
newobj_init(klass, flags, wb_protected, objspace, obj);
if (RB_UNLIKELY(ruby_gc_stressful)) {
rb_heap_t *heap = &heaps[heap_idx];
heap->newobj.alloc_cursor_end = heap->newobj.alloc_cursor;
}
return obj;
}
NOINLINE(static VALUE newobj_slowpath_wb_protected(VALUE klass, VALUE flags,
rb_objspace_t *objspace, size_t heap_idx));
NOINLINE(static VALUE newobj_slowpath_wb_unprotected(VALUE klass, VALUE flags,
rb_objspace_t *objspace, size_t heap_idx));
static VALUE
newobj_slowpath_wb_protected(VALUE klass, VALUE flags, rb_objspace_t *objspace, size_t heap_idx)
{
return newobj_slowpath(klass, flags, objspace, TRUE, heap_idx);
}
static VALUE
newobj_slowpath_wb_unprotected(VALUE klass, VALUE flags, rb_objspace_t *objspace, size_t heap_idx)
{
return newobj_slowpath(klass, flags, objspace, FALSE, heap_idx);
}
VALUE
rb_gc_impl_new_obj(void *objspace_ptr, void *cache_ptr, VALUE klass, VALUE flags, bool wb_protected, size_t alloc_size, size_t *actual_alloc_size)
{
VALUE obj;
rb_objspace_t *objspace = objspace_ptr;
/* There is no per-Ractor cache; the argument stays for ABI compatibility with other
* GC implementations such as MMTk. */
(void)cache_ptr;
RB_DEBUG_COUNTER_INC(obj_newobj);
(void)RB_DEBUG_COUNTER_INC_IF(obj_newobj_wb_unprotected, !wb_protected);
if (RB_UNLIKELY(stress_to_class)) {
if (rb_hash_lookup2(stress_to_class, klass, Qundef) != Qundef) {
rb_memerror();
}
}
size_t heap_idx = heap_idx_for_size(alloc_size);
*actual_alloc_size = heap_slot_size((unsigned char)heap_idx);
if (!RB_UNLIKELY(during_gc || ruby_gc_stressful) &&
wb_protected) {
obj = newobj_alloc(objspace, heap_idx);
newobj_init(klass, flags, wb_protected, objspace, obj);
}
else {
RB_DEBUG_COUNTER_INC(obj_newobj_slowpath);
obj = wb_protected ?
newobj_slowpath_wb_protected(klass, flags, objspace, heap_idx) :
newobj_slowpath_wb_unprotected(klass, flags, objspace, heap_idx);
}
return obj;
}
static int
ptr_in_page_body_p(const void *ptr, const void *memb)
{
struct heap_page *page = *(struct heap_page **)memb;
uintptr_t p_body = (uintptr_t)page->body;
if ((uintptr_t)ptr >= p_body) {
return (uintptr_t)ptr < (p_body + HEAP_PAGE_SIZE) ? 0 : 1;
}
else {
return -1;
}
}
PUREFUNC(static inline struct heap_page *heap_page_for_ptr(rb_objspace_t *objspace, uintptr_t ptr);)
static inline struct heap_page *
heap_page_for_ptr(rb_objspace_t *objspace, uintptr_t ptr)
{
struct heap_page **res;
if (ptr < (uintptr_t)heap_pages_lomem ||
ptr > (uintptr_t)heap_pages_himem) {
return NULL;
}
res = bsearch((void *)ptr, rb_darray_ref(objspace->heap_pages.sorted, 0),
rb_darray_size(objspace->heap_pages.sorted), sizeof(struct heap_page *),
ptr_in_page_body_p);
if (res) {
return *res;
}
else {
return NULL;
}
}
PUREFUNC(static inline bool is_pointer_to_heap(rb_objspace_t *objspace, const void *ptr);)
static inline bool
is_pointer_to_heap(rb_objspace_t *objspace, const void *ptr)
{
register uintptr_t p = (uintptr_t)ptr;
register struct heap_page *page;
RB_DEBUG_COUNTER_INC(gc_isptr_trial);
if (p < heap_pages_lomem || p > heap_pages_himem) return FALSE;
RB_DEBUG_COUNTER_INC(gc_isptr_range);
if (p % sizeof(VALUE) != 0) return FALSE;
RB_DEBUG_COUNTER_INC(gc_isptr_align);
page = heap_page_for_ptr(objspace, (uintptr_t)ptr);
if (page) {
RB_DEBUG_COUNTER_INC(gc_isptr_maybe);
if (heap_page_in_global_empty_pages_pool(objspace, page)) {
return FALSE;
}
else {
if (p < page->start) return FALSE;
if (p >= page->start + (page->total_slots * page->slot_size)) return FALSE;
if ((p - page->start) % page->slot_size != 0) return FALSE;
return TRUE;
}
}
return FALSE;
}
/*
* Is this object live in the given objspace. For foreign objects in other object spaces,
* it returns `false`. NOTE: this does NOT check whether or not it's unmarked during the process
* of lazy sweeping.
*/
bool
rb_gc_impl_live_object_p(void *objspace_ptr, const void *ptr)
{
rb_objspace_t *objspace = objspace_ptr;
/* Whether ptr refers to a live object. is_pointer_to_heap is the
* address-only check; T_NONE, T_MOVED, and T_ZOMBIE slots are valid heap
* addresses but not live objects. */
if (!is_pointer_to_heap(objspace, ptr)) return false;
VALUE obj = (VALUE)ptr;
bool live = false;
asan_unpoisoning_object(obj) {
switch (BUILTIN_TYPE(obj)) {
case T_NONE:
case T_MOVED:
case T_ZOMBIE:
break;
default:
live = true;
break;
}
}
return live;
}
/* Flags preserved from the original object when it becomes a zombie, and so also the
* only ones that may legitimately be set on one. */
#define ZOMBIE_OBJ_KEPT_FLAGS (FL_FINALIZE)
void
rb_gc_impl_make_zombie(void *objspace_ptr, VALUE obj, void (*dfree)(void *), void *data)
{
rb_objspace_t *objspace = objspace_ptr;
struct RZombie *zombie = RZOMBIE(obj);
zombie->flags = T_ZOMBIE | (zombie->flags & ZOMBIE_OBJ_KEPT_FLAGS);
zombie->dfree = dfree;
zombie->data = data;
VALUE prev, next = heap_pages_deferred_final;
do {
zombie->next = prev = next;
next = RUBY_ATOMIC_VALUE_CAS(heap_pages_deferred_final, prev, obj);
} while (next != prev);
struct heap_page *page = GET_HEAP_PAGE(obj);
page->final_slots++;
page->heap->final_slots_count++;
}
static void
tdata_unsafe_free_chunk_reset(struct tdata_unsafe_free_chunk *chunk)
{
chunk->next = NULL;
chunk->count = 0;
chunk->embed_xfree_bits = 0;
}
static struct tdata_unsafe_free_chunk *
tdata_unsafe_free_chunk_alloc(void)
{
/* Pops race each other (several Ractors can be sweeping), but pushes happen only
* inside the drain, which holds a VM barrier -- and a barrier cannot complete while
* a Ractor is inside gc_sweep_page. No push ever overlaps a pop, so the head only
* moves forward and this CAS pop needs no ABA tagging. A sweep performed by a
* thread other than the objspace's owner would break that. */
struct tdata_unsafe_free_chunk *head =
rbimpl_atomic_ptr_load((void **)&global_objspace->tdata_unsafe_free_cache,
RBIMPL_ATOMIC_ACQUIRE);
while (head) {
struct tdata_unsafe_free_chunk *prev =
rbimpl_atomic_ptr_cas((void **)&global_objspace->tdata_unsafe_free_cache,
head, head->next,
RBIMPL_ATOMIC_ACQ_REL, RBIMPL_ATOMIC_ACQUIRE);
if (prev == head) {
rbimpl_atomic_size_dec(&global_objspace->tdata_unsafe_free_cache_len,
RBIMPL_ATOMIC_RELAXED);
tdata_unsafe_free_chunk_reset(head);
return head;
}
head = prev;
}
/* Not xmalloc: this runs mid-sweep, and the chunks are GC bookkeeping that should not
* feed back into malloc_increase (mark stack chunks do the same). */
struct tdata_unsafe_free_chunk *chunk = malloc(sizeof(struct tdata_unsafe_free_chunk));
if (!chunk) rb_memerror();
tdata_unsafe_free_chunk_reset(chunk);
return chunk;
}
/* Hand this objspace's partial chunk to the global stack. The entries were counted as
* they were appended, so the pending count does not change here. */
static void
gc_tdata_unsafe_free_publish(rb_objspace_t *objspace)
{
struct tdata_unsafe_free_chunk *chunk = objspace->tdata_unsafe_free_chunk;
if (chunk == NULL) return;
GC_ASSERT(chunk->count > 0);
objspace->tdata_unsafe_free_chunk = NULL;
struct tdata_unsafe_free_chunk *prev, *head =
rbimpl_atomic_ptr_load((void **)&global_objspace->tdata_unsafe_free_published,
RBIMPL_ATOMIC_RELAXED);
do {
chunk->next = prev = head;
head = rbimpl_atomic_ptr_cas((void **)&global_objspace->tdata_unsafe_free_published,
prev, chunk,
RBIMPL_ATOMIC_ACQ_REL, RBIMPL_ATOMIC_ACQUIRE);
} while (head != prev);
}
/* Copy out what obj's deferred free needs, running no dfree. Returns true when the
* caller may reclaim the slot and false when obj became a zombie, matching rb_gc_obj_free. */
static bool
gc_defer_thread_unsafe_free(rb_objspace_t *objspace, VALUE obj, bool *trigger)
{
GC_ASSERT(!((uintptr_t)RTYPEDDATA(obj)->type & TYPED_DATA_EMBEDDED));
const rb_data_type_t *type = RTYPEDDATA_TYPE(obj);
void *data = RTYPEDDATA(obj)->data;
GC_ASSERT(data != NULL);
rb_gc_obj_free_vm_weak_references(obj);
size_t count = rbimpl_atomic_size_fetch_add(&global_objspace->tdata_deferred_free_count, 1,
RBIMPL_ATOMIC_RELAXED) + 1;
if (!*trigger && count >= TDATA_DEFERRED_FREE_THRESHOLD) {
*trigger = true;
}
struct tdata_unsafe_free_chunk *chunk = objspace->tdata_unsafe_free_chunk;
if (chunk == NULL) {
chunk = objspace->tdata_unsafe_free_chunk = tdata_unsafe_free_chunk_alloc();
}
if (type->flags & RUBY_TYPED_EMBEDDABLE) {
chunk->embed_xfree_bits |= (uint32_t)1 << chunk->count;
}
struct tdata_unsafe_free_entry *entry = &chunk->entries[chunk->count++];
entry->dfree = type->function.dfree;
entry->data = data;
if (chunk->count == TDATA_UNSAFE_FREE_CHUNK_CAPA) {
gc_tdata_unsafe_free_publish(objspace);
}
if (FL_TEST_RAW(obj, FL_FINALIZE)) {
/* The dfree is on the side list now, so this zombie carries none: it goes on the
* regular deferred list, where the owner runs its Ruby finalizer promptly and
* reclaims the slot, instead of waiting for the barrier. */
rb_gc_impl_make_zombie(objspace, obj, 0, 0);
return false;
}
return true;
}
typedef int each_obj_callback(void *, void *, size_t, void *);
typedef int each_page_callback(struct heap_page *, void *);
struct each_obj_data {
rb_objspace_t *objspace;
bool reenable_incremental;
/* Visit only the pages that hold shareable objects, so a foreign Ractor's objspace
* can be walked for its shareable objects alone, without touching the rest of its
* isolated heap. */
bool shareable_only;
/* Set when walking a foreign objspace without settling its stopped lazy sweep
* (settling would run the owner's obj_free and dfree on this thread). Objects the
* sweep is about to free are skipped: on an unswept page, unmarked means dead. */
bool skip_unswept_dead;
each_obj_callback *each_obj_callback;
each_page_callback *each_page_callback;
void *data;
struct heap_page **pages[HEAP_COUNT];
size_t pages_counts[HEAP_COUNT];
};
static VALUE
objspace_each_objects_ensure(VALUE arg)
{
struct each_obj_data *data = (struct each_obj_data *)arg;
rb_objspace_t *objspace = data->objspace;
/* Reenable incremental GC */
if (data->reenable_incremental) {
objspace->flags.dont_incremental = FALSE;
}
for (int i = 0; i < HEAP_COUNT; i++) {
struct heap_page **pages = data->pages[i];
free(pages);
}
return Qnil;
}
static VALUE
objspace_each_objects_try(VALUE arg)
{
struct each_obj_data *data = (struct each_obj_data *)arg;
rb_objspace_t *objspace = data->objspace;
/* Copy pages from all heaps to their respective buffers. */
for (int i = 0; i < HEAP_COUNT; i++) {
rb_heap_t *heap = &heaps[i];
size_t size = heap->total_pages * sizeof(struct heap_page *);
struct heap_page **pages = malloc(size);
if (!pages) rb_memerror();
/* Set up pages buffer by iterating over all pages in the current eden
* heap. This will be a snapshot of the state of the heap before we
* call the callback over each page that exists in this buffer. Thus it
* is safe for the callback to allocate objects without possibly entering
* an infinite loop. */
struct heap_page *page = 0;
size_t pages_count = 0;
ccan_list_for_each(&heap->pages, page, page_node) {
pages[pages_count] = page;
pages_count++;
}
data->pages[i] = pages;
data->pages_counts[i] = pages_count;
GC_ASSERT(pages_count == heap->total_pages);
}
for (int i = 0; i < HEAP_COUNT; i++) {
rb_heap_t *heap = &heaps[i];
size_t pages_count = data->pages_counts[i];
struct heap_page **pages = data->pages[i];
struct heap_page *page = ccan_list_top(&heap->pages, struct heap_page, page_node);
for (size_t i = 0; i < pages_count; i++) {
/* If we have reached the end of the linked list then there are no
* more pages, so break. */
if (page == NULL) break;
/* If this page does not match the one in the buffer, then move to
* the next page in the buffer. */
if (pages[i] != page) continue;
uintptr_t pstart = (uintptr_t)page->start;
uintptr_t pend = pstart + (page->total_slots * heap->slot_size);
if (data->shareable_only) {
/* Hand shareable objects to the callback one slot at a time, not the
* whole page: walking a foreign Ractor's objspace must never expose its
* unshareable objects, which the caller cannot inspect safely. */
if (page->flags.has_shareable_objects) {
/* This walk runs over a foreign objspace under the barrier and
* must not settle the owner's stopped lazy sweep: settling would run
* the owner's obj_free and dfree on this thread with this Ractor's
* identity (wrong per-Ractor tables, a foreign T_DATA dfree). So no
* gc_rest, and objects the sweep is about to free are skipped: on an
* unswept page unmarked means dead and its shareable bit merely has
* not been bulk-cleared yet. Passing one to the callback would
* resurrect it, handing out a reference the owner's sweep frees as
* soon as the barrier lifts. */
const bool page_unswept = is_lazy_sweeping(objspace) && page->flags.before_sweep;
int planes = CEILDIV(page->total_slots, BITS_BITLENGTH);
uintptr_t base = pstart;
bool stop = false;
for (int j = 0; j < planes && !stop; j++) {
bits_t bits = page->shareable_bits[j];
uintptr_t slot = base;
while (bits) {
if ((bits & 1) && data->each_obj_callback &&
!(page_unswept && !RVALUE_MARKED(objspace, (VALUE)slot)) &&
(*data->each_obj_callback)((void *)slot, (void *)(slot + heap->slot_size),
heap->slot_size, data->data)) {
stop = true;
break;
}
slot += heap->slot_size;
bits >>= 1;
}
base += BITS_BITLENGTH * heap->slot_size;
}
if (stop) break;
}
}
else if (data->skip_unswept_dead &&
is_lazy_sweeping(objspace) && page->flags.before_sweep) {
/* A foreign page pending sweep: hand out the live objects one slot at a
* time and skip the unmarked (dead) ones the owner's sweep frees as soon
* as the barrier lifts. */
bool stop = false;
for (uintptr_t slot = pstart; slot < pend; slot += heap->slot_size) {
if (!RVALUE_MARKED(objspace, (VALUE)slot)) continue;
if (data->each_obj_callback &&
(*data->each_obj_callback)((void *)slot, (void *)(slot + heap->slot_size),
heap->slot_size, data->data)) {
stop = true;
break;
}
}
if (stop) break;
}
else {
if (data->each_obj_callback &&
(*data->each_obj_callback)((void *)pstart, (void *)pend, heap->slot_size, data->data)) {
break;
}
if (data->each_page_callback &&
(*data->each_page_callback)(page, data->data)) {
break;
}
}
page = ccan_list_next(&heap->pages, page, page_node);
}
}
return Qnil;
}
static void
objspace_each_exec(bool protected, struct each_obj_data *each_obj_data)
{
/* Disable incremental GC */
rb_objspace_t *objspace = each_obj_data->objspace;
bool reenable_incremental = FALSE;
if (protected) {
reenable_incremental = !objspace->flags.dont_incremental;
gc_rest(objspace);
objspace->flags.dont_incremental = TRUE;
}
each_obj_data->reenable_incremental = reenable_incremental;
memset(&each_obj_data->pages, 0, sizeof(each_obj_data->pages));
memset(&each_obj_data->pages_counts, 0, sizeof(each_obj_data->pages_counts));
rb_ensure(objspace_each_objects_try, (VALUE)each_obj_data,
objspace_each_objects_ensure, (VALUE)each_obj_data);
}
static void
objspace_each_objects(rb_objspace_t *objspace, each_obj_callback *callback, void *data, bool protected)
{
struct each_obj_data each_obj_data = {
.objspace = objspace,
.each_obj_callback = callback,
.each_page_callback = NULL,
.data = data,
};
objspace_each_exec(protected, &each_obj_data);
}
void
rb_gc_impl_each_objects(void *objspace_ptr, each_obj_callback *callback, void *data)
{
objspace_each_objects(objspace_ptr, callback, data, TRUE);
}
/* Like rb_gc_impl_each_objects but visiting only pages that hold shareable objects, to
* reach a foreign Ractor's shareable objects without walking the rest of its heap. */
void
rb_gc_impl_each_objects_shareable(void *objspace_ptr, each_obj_callback *callback, void *data)
{
struct each_obj_data each_obj_data = {
.objspace = objspace_ptr,
.shareable_only = true,
.each_obj_callback = callback,
.each_page_callback = NULL,
.data = data,
};
/* Not the protected variant: this objspace belongs to another Ractor (the caller
* holds the barrier). The protected path calls gc_rest, which would run the owner's
* stopped lazy sweep (its obj_free and dfree) on the walking thread with the
* walker's Ractor identity (wrong per-Ractor tables, a foreign T_DATA dfree). The
* owner is stopped and its page list is stable, and the walk itself skips dead,
* unswept objects (the shareable_only branch of objspace_each_objects_try). The
* walker's own incremental GC state is untouched, since this is not its objspace. */
objspace_each_exec(FALSE, &each_obj_data);
}
/* Walk every object of a foreign Ractor's objspace, unshareable ones included. Only for
* callers that hold the barrier and whose callback is pure C (a heap dump, memory
* accounting). As in the shareable walk above, the owner's stopped lazy sweep is not
* settled and dead, unswept objects are skipped by the walk (skip_unswept_dead). */
void
rb_gc_impl_each_objects_foreign(void *objspace_ptr, each_obj_callback *callback, void *data)
{
struct each_obj_data each_obj_data = {
.objspace = objspace_ptr,
.skip_unswept_dead = true,
.each_obj_callback = callback,
.each_page_callback = NULL,
.data = data,
};
objspace_each_exec(FALSE, &each_obj_data);
}
#if GC_CAN_COMPILE_COMPACTION
static void
objspace_each_pages(rb_objspace_t *objspace, each_page_callback *callback, void *data, bool protected)
{
struct each_obj_data each_obj_data = {
.objspace = objspace,
.each_obj_callback = NULL,
.each_page_callback = callback,
.data = data,
};
objspace_each_exec(protected, &each_obj_data);
}
#endif
VALUE
rb_gc_impl_define_finalizer(void *objspace_ptr, VALUE obj, VALUE block)
{
rb_objspace_t *objspace = objspace_ptr;
VALUE table;
st_data_t data;
GC_ASSERT(!OBJ_FROZEN(obj));
/* Registering, storing and running finalizers all belong to the object's own
* objspace, so refuse to define one on another Ractor's object (even a shareable
* one): it would land in a table the owner's sweep never consults. */
if (GET_HEAP_OBJSPACE(obj) != objspace) {
rb_raise(rb_eRactorIsolationError,
"can not define a finalizer for an object of another Ractor");
}
RBASIC(obj)->flags |= FL_FINALIZE;
unsigned int lev = RB_GC_VM_LOCK();
if (st_lookup(finalizer_table, obj, &data)) {
table = (VALUE)data;
VALUE dup_table = rb_ary_dup(table);
RB_GC_VM_UNLOCK(lev);
/* avoid duplicate block, table is usually small */
{
rb_len_t len = RARRAY_LEN(table);
rb_len_t i;
for (i = 0; i < len; i++) {
VALUE recv = RARRAY_AREF(dup_table, i);
if (rb_equal(recv, block)) { // can't be called with VM lock held
return recv;
}
}
}
lev = RB_GC_VM_LOCK();
RB_GC_GUARD(dup_table);
rb_ary_push(table, block);
}
else {
table = rb_ary_new3(2, rb_obj_id(obj), block);
rb_obj_hide(table);
st_add_direct(finalizer_table, obj, table);
}
RB_GC_VM_UNLOCK(lev);
return block;
}
void
rb_gc_impl_undefine_finalizer(void *objspace_ptr, VALUE obj)
{
rb_objspace_t *objspace = objspace_ptr;
GC_ASSERT(!OBJ_FROZEN(obj));
/* Symmetric with define. */
if (GET_HEAP_OBJSPACE(obj) != objspace) {
rb_raise(rb_eRactorIsolationError,
"can not undefine a finalizer of an object of another Ractor");
}
st_data_t data = obj;
int lev = RB_GC_VM_LOCK();
st_delete(finalizer_table, &data, 0);
RB_GC_VM_UNLOCK(lev);
FL_UNSET(obj, FL_FINALIZE);
}
void
rb_gc_impl_copy_finalizer(void *objspace_ptr, VALUE dest, VALUE obj)
{
/* Finalizers do not cross objspaces: a copy of another Ractor's object starts with
* none (guards the public rb_gc_copy_finalizer C API; no in-tree caller crosses).
* A same-objspace copy behaves as before. Table accessed under the VM lock. */
rb_objspace_t *objspace = objspace_ptr;
VALUE table;
st_data_t data;
if (!FL_TEST(obj, FL_FINALIZE)) return;
if (GET_HEAP_OBJSPACE(obj) != objspace) return;
int lev = RB_GC_VM_LOCK();
if (RB_LIKELY(st_lookup(finalizer_table, obj, &data))) {
table = rb_ary_dup((VALUE)data);
RARRAY_ASET(table, 0, rb_obj_id(dest));
st_insert(finalizer_table, dest, table);
FL_SET(dest, FL_FINALIZE);
}
else {
rb_bug("rb_gc_copy_finalizer: FL_FINALIZE set but not found in finalizer_table: %s", rb_obj_info(obj));
}
RB_GC_VM_UNLOCK(lev);
}
static VALUE
get_final(rb_len_t i, void *data)
{
VALUE table = (VALUE)data;
return RARRAY_AREF(table, i + 1);
}
static void
run_final(rb_objspace_t *objspace, VALUE zombie)
{
if (RZOMBIE(zombie)->dfree) {
RZOMBIE(zombie)->dfree(RZOMBIE(zombie)->data);
}
st_data_t key = (st_data_t)zombie;
if (FL_TEST_RAW(zombie, FL_FINALIZE)) {
FL_UNSET(zombie, FL_FINALIZE);
st_data_t table;
if (st_delete(finalizer_table, &key, &table)) {
rb_gc_run_obj_finalizer(RARRAY_AREF(table, 0), RARRAY_LEN(table) - 1, get_final, (void *)table);
}
else {
rb_bug("FL_FINALIZE flag is set, but finalizers are not found");
}
}
else {
GC_ASSERT(!st_lookup(finalizer_table, key, NULL));
}
}
static void
finalize_list(rb_objspace_t *objspace, VALUE zombie)
{
while (zombie) {
VALUE next_zombie;
struct heap_page *page;
rb_asan_unpoison_object(zombie, false);
next_zombie = RZOMBIE(zombie)->next;
page = GET_HEAP_PAGE(zombie);
run_final(objspace, zombie);
{
GC_ASSERT(BUILTIN_TYPE(zombie) == T_ZOMBIE);
GC_ASSERT(page->heap->final_slots_count > 0);
GC_ASSERT(page->final_slots > 0);
page->heap->final_slots_count--;
page->final_slots--;
page->free_slots++;
RVALUE_AGE_SET_BITMAP(zombie, 0);
heap_page_add_free_region(objspace, page, zombie);
page->heap->total_freed_objects++;
}
zombie = next_zombie;
}
}
static void
finalize_zombies(rb_objspace_t *objspace)
{
VALUE zombie;
while ((zombie = RUBY_ATOMIC_VALUE_EXCHANGE(heap_pages_deferred_final, 0)) != 0) {
finalize_list(objspace, zombie);
}
}
static void
finalize_deferred(rb_objspace_t *objspace)
{
rb_gc_set_pending_interrupt();
finalize_zombies(objspace);
rb_gc_unset_pending_interrupt();
}
static void
gc_finalize_deferred(void *dmy)
{
/* One postponed job is shared by every objspace: the preregistration table only
* holds about 32 entries and Ractors are created continuously. A deferred finalizer
* belongs to the objspace of the thread that ran the job, i.e. the current one. */
rb_objspace_t *objspace = rb_gc_get_objspace();
if (RUBY_ATOMIC_EXCHANGE(finalizing, 1)) return;
finalize_deferred(objspace);
RUBY_ATOMIC_SET(finalizing, 0);
}
static void
gc_finalize_deferred_register(rb_objspace_t *objspace)
{
/* Enqueue gc_finalize_deferred on this objspace's owning Ractor. A global GC can
* defer a foreign objspace's finalizers, and those must run on their owner rather
* than on the driver. */
rb_gc_trigger_finalize_deferred(objspace, objspace->finalize_deferred_pjob);
}
static int pop_mark_stack(mark_stack_t *stack, VALUE *data);
/* Throw away an unfinished incremental mark and leave the objspace in gc_mode_none. The
* mark bits the partial mark set stay behind, so the caller must clear them before the
* heap is collected again. */
static void
gc_abort_incremental_marking(rb_objspace_t *objspace)
{
GC_ASSERT(is_incremental_marking(objspace));
VALUE obj;
while (pop_mark_stack(&objspace->mark_stack, &obj));
/* gc_grey records the weak references it greys for gc_marks_finish to resolve; this
* cycle never reaches it, and the entries would outlive their objects. */
rb_darray_clear(objspace->weak_references);
gc_set_incremental_marking(objspace, FALSE);
gc_mode_set(objspace, gc_mode_none);
}
static void
gc_abort(void *objspace_ptr)
{
rb_objspace_t *objspace = objspace_ptr;
if (is_incremental_marking(objspace)) {
gc_abort_incremental_marking(objspace);
}
if (is_lazy_sweeping(objspace)) {
objspace->sweeping_heap_count = 0;
for (int i = 0; i < HEAP_COUNT; i++) {
rb_heap_t *heap = &heaps[i];
heap->sweeping_page = NULL;
struct heap_page *page = NULL;
ccan_list_for_each(&heap->pages, page, page_node) {
page->flags.before_sweep = false;
}
}
}
for (int i = 0; i < HEAP_COUNT; i++) {
rb_heap_t *heap = &heaps[i];
gc_bitmaps_clear(objspace, heap, false);
}
gc_mode_set(objspace, gc_mode_none);
}
#if VERIFY_FREE_SIZE
# ifdef RB_THREAD_LOCAL_SPECIFIER
# define GC_FREEING_OBJ_TLS RB_THREAD_LOCAL_SPECIFIER
# else
# define GC_FREEING_OBJ_TLS
# endif
static GC_FREEING_OBJ_TLS VALUE gc_freeing_obj;
/* Remember what we are tearing down so that a bad xfree() underneath can name
* the object and not just the buffer. Saved and restored because a dfree
* callback can free another object. */
static bool
gc_obj_free(void *objspace, VALUE obj)
{
VALUE prev = gc_freeing_obj;
gc_freeing_obj = obj;
bool freed = rb_gc_obj_free(objspace, obj);
gc_freeing_obj = prev;
return freed;
}
static const char *
gc_freeing_obj_info(void)
{
/* Not thread-local: only reachable from a rb_bug() path, where a second
* thread racing us is already unrecoverable. */
static char buf[128];
if (!gc_freeing_obj) return NULL;
snprintf(buf, sizeof(buf), "%p %s", (void *)gc_freeing_obj, rb_obj_info(gc_freeing_obj));
return buf;
}
#else
# define gc_obj_free(objspace, obj) rb_gc_obj_free((objspace), (obj))
# define gc_freeing_obj_info() NULL
#endif
void
rb_gc_impl_shutdown_free_objects(void *objspace_ptr)
{
rb_objspace_t *objspace = objspace_ptr;
for (size_t i = 0; i < rb_darray_size(objspace->heap_pages.sorted); i++) {
struct heap_page *page = rb_darray_get(objspace->heap_pages.sorted, i);
short stride = page->slot_size;
uintptr_t p = (uintptr_t)page->start;
uintptr_t pend = p + page->total_slots * stride;
for (; p < pend; p += stride) {
VALUE vp = (VALUE)p;
asan_unpoisoning_object(vp) {
if (RB_BUILTIN_TYPE(vp) != T_NONE) {
rb_gc_obj_free_vm_weak_references(vp);
if (gc_obj_free(objspace, vp)) {
RBASIC(vp)->flags = 0;
}
}
}
}
}
}
static int
rb_gc_impl_shutdown_call_finalizer_i(st_data_t key, st_data_t val, st_data_t _data)
{
VALUE obj = (VALUE)key;
VALUE table = (VALUE)val;
GC_ASSERT(RB_FL_TEST(obj, FL_FINALIZE));
GC_ASSERT(RB_BUILTIN_TYPE(val) == T_ARRAY);
rb_gc_run_obj_finalizer(RARRAY_AREF(table, 0), RARRAY_LEN(table) - 1, get_final, (void *)table);
/* obj may be a poisoned zombie */
asan_unpoisoning_object(obj) {
FL_UNSET(obj, FL_FINALIZE);
}
return ST_DELETE;
}
void
rb_gc_impl_shutdown_call_finalizer(void *objspace_ptr)
{
rb_objspace_t *objspace = objspace_ptr;
#if RGENGC_CHECK_MODE >= 2
gc_verify_internal_consistency(objspace);
#endif
/* prohibit incremental GC */
objspace->flags.dont_incremental = 1;
if (RUBY_ATOMIC_EXCHANGE(finalizing, 1)) {
/* Abort incremental marking and lazy sweeping to speed up shutdown. */
gc_abort(objspace);
dont_gc_on();
return;
}
while (finalizer_table->num_entries) {
st_foreach(finalizer_table, rb_gc_impl_shutdown_call_finalizer_i, 0);
}
/* run finalizers */
finalize_deferred(objspace);
GC_ASSERT(heap_pages_deferred_final == 0);
/* Deferred non-thread-safe frees: their objects are long gone, so the object walk
* below will not reach them. Reap them here. */
gc_tdata_unsafe_drain_objspaces(&objspace, 1);
/* Abort incremental marking and lazy sweeping to speed up shutdown. */
gc_abort(objspace);
/* prohibit GC because force T_DATA finalizers can break an object graph consistency */
dont_gc_on();
/* running data/file finalizers are part of garbage collection */
unsigned int lock_lev;
gc_enter(objspace, gc_enter_event_finalizer, &lock_lev);
/* run data/file object's finalizers */
for (size_t i = 0; i < rb_darray_size(objspace->heap_pages.sorted); i++) {
struct heap_page *page = rb_darray_get(objspace->heap_pages.sorted, i);
short stride = page->slot_size;
uintptr_t p = (uintptr_t)page->start;
uintptr_t pend = p + page->total_slots * stride;
for (; p < pend; p += stride) {
VALUE vp = (VALUE)p;
asan_unpoisoning_object(vp) {
if (rb_gc_shutdown_call_finalizer_p(vp)) {
rb_gc_obj_free_vm_weak_references(vp);
if (gc_obj_free(objspace, vp)) {
RBASIC(vp)->flags = 0;
}
}
}
}
}
gc_exit(objspace, gc_enter_event_finalizer, &lock_lev);
finalize_zombies(objspace);
st_free_table(finalizer_table);
finalizer_table = 0;
RUBY_ATOMIC_SET(finalizing, 0);
}
void
rb_gc_impl_each_object(void *objspace_ptr, void (*func)(VALUE obj, void *data), void *data)
{
rb_objspace_t *objspace = objspace_ptr;
for (size_t i = 0; i < rb_darray_size(objspace->heap_pages.sorted); i++) {
struct heap_page *page = rb_darray_get(objspace->heap_pages.sorted, i);
short stride = page->slot_size;
uintptr_t p = (uintptr_t)page->start;
uintptr_t pend = p + page->total_slots * stride;
for (; p < pend; p += stride) {
VALUE obj = (VALUE)p;
asan_unpoisoning_object(obj) {
func(obj, data);
}
}
}
}
/*
------------------------ Garbage Collection ------------------------
*/
/* Sweeping */
static size_t
objspace_available_slots(rb_objspace_t *objspace)
{
size_t total_slots = 0;
for (int i = 0; i < HEAP_COUNT; i++) {
rb_heap_t *heap = &heaps[i];
total_slots += heap->total_slots;
}
return total_slots;
}
static size_t
objspace_live_slots(rb_objspace_t *objspace)
{
return total_allocated_objects(objspace) - total_freed_objects(objspace) - total_final_slots_count(objspace);
}
static size_t
objspace_free_slots(rb_objspace_t *objspace)
{
return objspace_available_slots(objspace) - objspace_live_slots(objspace) - total_final_slots_count(objspace);
}
static void
gc_setup_mark_bits(struct heap_page *page)
{
/* copy oldgen bitmap to mark bitmap */
memcpy(&page->mark_bits[0], &page->uncollectible_bits[0], HEAP_PAGE_BITMAP_SIZE);
}
static int gc_is_moveable_obj(rb_objspace_t *objspace, VALUE obj);
static VALUE gc_move(rb_objspace_t *objspace, VALUE scan, VALUE free, struct heap_page *src_page, struct heap_page *dest_page);
#if defined(_WIN32)
enum {HEAP_PAGE_LOCK = PAGE_NOACCESS, HEAP_PAGE_UNLOCK = PAGE_READWRITE};
static BOOL
protect_page_body(struct heap_page_body *body, DWORD protect)
{
DWORD old_protect;
return VirtualProtect(body, HEAP_PAGE_SIZE, protect, &old_protect) != 0;
}
#elif defined(__wasi__)
// wasi-libc's mprotect emulation does not support PROT_NONE
enum {HEAP_PAGE_LOCK, HEAP_PAGE_UNLOCK};
#define protect_page_body(body, protect) 1
#else
enum {HEAP_PAGE_LOCK = PROT_NONE, HEAP_PAGE_UNLOCK = PROT_READ | PROT_WRITE};
#define protect_page_body(body, protect) !mprotect((body), HEAP_PAGE_SIZE, (protect))
#endif
static void
lock_page_body(rb_objspace_t *objspace, struct heap_page_body *body)
{
if (!protect_page_body(body, HEAP_PAGE_LOCK)) {
rb_bug("Couldn't protect page %p, errno: %s", (void *)body, strerror(errno));
}
else {
gc_report(5, objspace, "Protecting page in move %p\n", (void *)body);
}
}
static void
unlock_page_body(rb_objspace_t *objspace, struct heap_page_body *body)
{
if (!protect_page_body(body, HEAP_PAGE_UNLOCK)) {
rb_bug("Couldn't unprotect page %p, errno: %s", (void *)body, strerror(errno));
}
else {
gc_report(5, objspace, "Unprotecting page in move %p\n", (void *)body);
}
}
static uintptr_t
heap_page_alloc_slot_from_region(struct heap_page *free_page)
{
asan_unlock_freelist(free_page);
struct free_region *region = free_page->free_region;
asan_lock_freelist(free_page);
if (region == NULL) {
return 0;
}
rb_asan_unpoison_object((VALUE)region, false);
GC_ASSERT(RB_TYPE_P((VALUE)region, T_NONE));
uintptr_t dest = (uintptr_t)region;
uintptr_t region_end = region->end;
struct free_region *next = region->next;
uintptr_t new_start = dest + free_page->slot_size;
asan_unlock_freelist(free_page);
if (new_start < region_end) {
VALUE next_start = (VALUE)new_start;
rb_asan_unpoison_object(next_start, false);
struct free_region *new_region = (struct free_region *)new_start;
new_region->flags = 0;
new_region->end = region_end;
new_region->next = next;
rb_asan_poison_object(next_start);
free_page->free_region = new_region;
}
else {
free_page->free_region = next;
}
asan_lock_freelist(free_page);
return dest;
}
static bool
try_move(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *free_page, VALUE src)
{
GC_ASSERT(gc_is_moveable_obj(objspace, src));
struct heap_page *src_page = GET_HEAP_PAGE(src);
if (!free_page) {
return false;
}
/* We should return true if either src is successfully moved, or src is
* unmoveable. A false return will cause the sweeping cursor to be
* incremented to the next page, and src will attempt to move again */
GC_ASSERT(RVALUE_MARKED(objspace, src));
uintptr_t dest_slot = heap_page_alloc_slot_from_region(free_page);
if (dest_slot == 0) {
return false;
}
VALUE dest = (VALUE)dest_slot;
GC_ASSERT(RB_BUILTIN_TYPE(dest) == T_NONE);
if (src_page->slot_size > free_page->slot_size) {
objspace->rcompactor.moved_down_count_table[BUILTIN_TYPE(src)]++;
}
else if (free_page->slot_size > src_page->slot_size) {
objspace->rcompactor.moved_up_count_table[BUILTIN_TYPE(src)]++;
}
objspace->rcompactor.moved_count_table[BUILTIN_TYPE(src)]++;
objspace->rcompactor.total_moved++;
gc_move(objspace, src, dest, src_page, free_page);
gc_pin(objspace, src);
free_page->free_slots--;
return true;
}
static void
gc_unprotect_pages(rb_objspace_t *objspace, rb_heap_t *heap)
{
struct heap_page *cursor = heap->compact_cursor;
while (cursor) {
unlock_page_body(objspace, cursor->body);
cursor = ccan_list_next(&heap->pages, cursor, page_node);
}
}
static void gc_update_references(rb_objspace_t *objspace);
static void gc_update_references_heap(rb_objspace_t *objspace);
static void gc_update_references_global(rb_objspace_t *objspace);
#if GC_CAN_COMPILE_COMPACTION
static void invalidate_moved_page(rb_objspace_t *objspace, struct heap_page *page);
#endif
#if defined(__MINGW32__) || defined(_WIN32)
# define GC_COMPACTION_SUPPORTED 1
#else
/* If not MinGW, Windows, or does not have mmap, we cannot use mprotect for
* the read barrier, so we must disable compaction. */
# define GC_COMPACTION_SUPPORTED (GC_CAN_COMPILE_COMPACTION && HEAP_PAGE_ALLOC_USE_MMAP)
#endif
#if GC_CAN_COMPILE_COMPACTION
static void
read_barrier_handler(uintptr_t address)
{
rb_objspace_t *objspace = (rb_objspace_t *)rb_gc_get_objspace();
struct heap_page_body *page_body = GET_PAGE_BODY(address);
/* If the page_body is NULL, then mprotect cannot handle it and will crash
* with "Cannot allocate memory". */
if (page_body == NULL) {
rb_bug("read_barrier_handler: segmentation fault at %p", (void *)address);
}
int lev = RB_GC_VM_LOCK();
{
unlock_page_body(objspace, page_body);
objspace->profile.read_barrier_faults++;
invalidate_moved_page(objspace, GET_HEAP_PAGE(address));
}
RB_GC_VM_UNLOCK(lev);
}
#endif
#if !GC_CAN_COMPILE_COMPACTION
static void
uninstall_handlers(void)
{
/* no-op */
}
static void
install_handlers(void)
{
/* no-op */
}
#elif defined(_WIN32)
static LPTOP_LEVEL_EXCEPTION_FILTER old_handler;
typedef void (*signal_handler)(int);
static signal_handler old_sigsegv_handler;
static LONG WINAPI
read_barrier_signal(EXCEPTION_POINTERS *info)
{
/* EXCEPTION_ACCESS_VIOLATION is what's raised by access to protected pages */
if (info->ExceptionRecord->ExceptionCode == EXCEPTION_ACCESS_VIOLATION) {
/* > The second array element specifies the virtual address of the inaccessible data.
* https://docs.microsoft.com/en-us/windows/win32/api/winnt/ns-winnt-exception_record
*
* Use this address to invalidate the page */
read_barrier_handler((uintptr_t)info->ExceptionRecord->ExceptionInformation[1]);
return EXCEPTION_CONTINUE_EXECUTION;
}
else {
return EXCEPTION_CONTINUE_SEARCH;
}
}
static void
uninstall_handlers(void)
{
signal(SIGSEGV, old_sigsegv_handler);
SetUnhandledExceptionFilter(old_handler);
}
static void
install_handlers(void)
{
/* Remove SEGV handler so that the Unhandled Exception Filter handles it */
old_sigsegv_handler = signal(SIGSEGV, NULL);
/* Unhandled Exception Filter has access to the violation address similar
* to si_addr from sigaction */
old_handler = SetUnhandledExceptionFilter(read_barrier_signal);
}
#else
static struct sigaction old_sigbus_handler;
static struct sigaction old_sigsegv_handler;
#ifdef HAVE_MACH_TASK_EXCEPTION_PORTS
static exception_mask_t old_exception_masks[32];
static mach_port_t old_exception_ports[32];
static exception_behavior_t old_exception_behaviors[32];
static thread_state_flavor_t old_exception_flavors[32];
static mach_msg_type_number_t old_exception_count;
static void
disable_mach_bad_access_exc(void)
{
old_exception_count = sizeof(old_exception_masks) / sizeof(old_exception_masks[0]);
task_swap_exception_ports(
mach_task_self(), EXC_MASK_BAD_ACCESS,
MACH_PORT_NULL, EXCEPTION_DEFAULT, 0,
old_exception_masks, &old_exception_count,
old_exception_ports, old_exception_behaviors, old_exception_flavors
);
}
static void
restore_mach_bad_access_exc(void)
{
for (mach_msg_type_number_t i = 0; i < old_exception_count; i++) {
task_set_exception_ports(
mach_task_self(),
old_exception_masks[i], old_exception_ports[i],
old_exception_behaviors[i], old_exception_flavors[i]
);
}
}
#endif
#if defined(HAVE_PTHREAD_SIGMASK)
# define gc_sigmask pthread_sigmask
#else
# define gc_sigmask sigprocmask
#endif
static void
read_barrier_signal(int sig, siginfo_t *info, void *data)
{
// setup SEGV/BUS handlers for errors
struct sigaction prev_sigbus, prev_sigsegv;
sigaction(SIGBUS, &old_sigbus_handler, &prev_sigbus);
sigaction(SIGSEGV, &old_sigsegv_handler, &prev_sigsegv);
// enable SIGBUS/SEGV
sigset_t set, prev_set;
sigemptyset(&set);
sigaddset(&set, SIGBUS);
sigaddset(&set, SIGSEGV);
gc_sigmask(SIG_UNBLOCK, &set, &prev_set);
#ifdef HAVE_MACH_TASK_EXCEPTION_PORTS
disable_mach_bad_access_exc();
#endif
// run handler
read_barrier_handler((uintptr_t)info->si_addr);
// reset SEGV/BUS handlers
#ifdef HAVE_MACH_TASK_EXCEPTION_PORTS
restore_mach_bad_access_exc();
#endif
sigaction(SIGBUS, &prev_sigbus, NULL);
sigaction(SIGSEGV, &prev_sigsegv, NULL);
gc_sigmask(SIG_SETMASK, &prev_set, NULL);
}
static void
uninstall_handlers(void)
{
#ifdef HAVE_MACH_TASK_EXCEPTION_PORTS
restore_mach_bad_access_exc();
#endif
sigaction(SIGBUS, &old_sigbus_handler, NULL);
sigaction(SIGSEGV, &old_sigsegv_handler, NULL);
}
static void
install_handlers(void)
{
struct sigaction action;
memset(&action, 0, sizeof(struct sigaction));
sigemptyset(&action.sa_mask);
action.sa_sigaction = read_barrier_signal;
action.sa_flags = SA_SIGINFO | SA_ONSTACK;
sigaction(SIGBUS, &action, &old_sigbus_handler);
sigaction(SIGSEGV, &action, &old_sigsegv_handler);
#ifdef HAVE_MACH_TASK_EXCEPTION_PORTS
disable_mach_bad_access_exc();
#endif
}
#endif
static void
gc_compact_finish(rb_objspace_t *objspace)
{
for (int i = 0; i < HEAP_COUNT; i++) {
rb_heap_t *heap = &heaps[i];
gc_unprotect_pages(objspace, heap);
}
if (!global_objspace->global_gc.compacting) uninstall_handlers();
if (global_objspace->global_gc.compacting) {
/* In a compacting global GC this updates only this objspace's heap references;
* gc_start_global sets during_reference_updating on every objspace (the
* move-or-mark decision reads it via rb_gc_get_objspace()) and runs the
* non-idempotent VM-global side (gc_update_references_global) once at the end. */
gc_update_references_heap(objspace);
}
else {
gc_update_references(objspace);
}
objspace->profile.compact_count++;
for (int i = 0; i < HEAP_COUNT; i++) {
rb_heap_t *heap = &heaps[i];
heap->compact_cursor = NULL;
heap->free_pages = NULL;
heap->compact_cursor_index = 0;
}
if (gc_prof_enabled(objspace)) {
gc_profile_record *record = gc_prof_record(objspace);
record->moved_objects = objspace->rcompactor.total_moved - record->moved_objects;
}
if (!global_objspace->global_gc.compacting) objspace->flags.during_compacting = FALSE;
}
struct gc_sweep_context {
struct heap_page *page;
int final_slots;
int freed_slots;
int empty_slots;
/* Hoisted out of the per-slot pinned-free assert: too expensive for the sweep loop
* as an external call. */
const bool check_pinned_free;
/* This is a parallel local sweep (multi-Ractor, not a global GC), so a non-thread-safe
* T_DATA dfree must be deferred to the global GC or the postponed job rather than run here. */
const bool defer_thread_unsafe_local_sweep;
bool trigger_thread_unsafe_sweep_postponed_job;
struct free_region *free_region;
};
/* NOTE: We must free the root fiber during postmortem collection, otherwise another Ractor
* can collect the fiber through a major GC while we're still tearing it down. Once fibers are
* THREAD_SAFE_FREE, we no longer need the root fiber condition as it will be guaranteed to be
* collected during this time. */
static bool
gc_obj_defer_local_free_p(rb_objspace_t *objspace, VALUE obj)
{
if (BUILTIN_TYPE(obj) != T_DATA) return false;
const rb_data_type_t *type = RTYPEDDATA_TYPE(obj);
if (!rb_gc_data_type_deferred_free_p(type)) return false;
if (RTYPEDDATA_GET_DATA(obj) == NULL) return false;
if (type->flags & RUBY_TYPED_FREE_IMMEDIATELY) {
if (objspace->flags.during_postmortem) {
if (rb_fiber_current() == obj) {
return false;
}
}
return true;
}
else {
return false;
}
}
static void gc_tdata_deferred_free_job(void *unused);
static void gc_tdata_deferred_free_pjob_ensure(void);
static unsigned int gc_during_gc_get(const rb_objspace_t *objspace);
static void gc_during_gc_set(rb_objspace_t *objspace, unsigned int v);
static void gc_global_snapshot_objspaces(void);
static void
gc_tdata_deferred_free_pjob_ensure(void)
{
if (global_objspace->tdata_deferred_free_pjob == POSTPONED_JOB_HANDLE_INVALID) {
global_objspace->tdata_deferred_free_pjob =
rb_postponed_job_preregister(0, gc_tdata_deferred_free_job, NULL);
if (global_objspace->tdata_deferred_free_pjob == POSTPONED_JOB_HANDLE_INVALID) {
rb_bug("Could not preregister postponed job for deferred T_DATA free");
}
}
}
/* A terminating Ractor's postmortem collection runs on an EC whose stack is already
* torn down: it never checks interrupts again, so a job triggered there is lost and no
* later sweep can rediscover the entries. Hand those to the main Ractor. */
static void
gc_tdata_deferred_free_trigger(rb_objspace_t *objspace)
{
if (objspace->flags.during_postmortem) {
rb_gc_trigger_postponed_job_on_main(global_objspace->tdata_deferred_free_pjob);
}
else {
rb_postponed_job_trigger(global_objspace->tdata_deferred_free_pjob);
}
}
static void
gc_tdata_unsafe_free_entry(const struct tdata_unsafe_free_entry *entry, bool embed_xfree)
{
entry->dfree(entry->data);
if (embed_xfree) {
xfree(entry->data);
}
}
static void
tdata_unsafe_free_chunk_recycle(struct tdata_unsafe_free_chunk *chunk)
{
if (global_objspace->tdata_unsafe_free_cache_len >= TDATA_UNSAFE_FREE_CACHE_MAX) {
free(chunk);
return;
}
tdata_unsafe_free_chunk_reset(chunk);
chunk->next = global_objspace->tdata_unsafe_free_cache;
rbimpl_atomic_ptr_store((volatile void **)&global_objspace->tdata_unsafe_free_cache, chunk,
RBIMPL_ATOMIC_RELEASE);
global_objspace->tdata_unsafe_free_cache_len++;
}
static void
gc_tdata_unsafe_drain_chunk(struct tdata_unsafe_free_chunk *chunk)
{
for (unsigned int i = 0; i < chunk->count; i++) {
gc_tdata_unsafe_free_entry(&chunk->entries[i],
(chunk->embed_xfree_bits >> i) & 1);
}
tdata_unsafe_free_chunk_recycle(chunk);
}
/* Run every pending deferred free: the published chunks (which belong to no objspace)
* plus the given objspaces' partial chunks. The caller must have stopped the world --
* VM barrier held, or a single Ractor left in the process -- and must pass every live
* objspace, since the pending count is zeroed here. (Shutdown is the one exception:
* nothing reads the count afterwards.) */
static void
gc_tdata_unsafe_drain_objspaces(rb_objspace_t **objspaces, size_t n)
{
struct tdata_unsafe_free_chunk *chunk =
rbimpl_atomic_ptr_exchange((void **)&global_objspace->tdata_unsafe_free_published, NULL,
RBIMPL_ATOMIC_ACQ_REL);
while (chunk) {
struct tdata_unsafe_free_chunk *next = chunk->next;
gc_tdata_unsafe_drain_chunk(chunk);
chunk = next;
}
for (size_t i = 0; i < n; i++) {
rb_objspace_t *os = objspaces[i];
struct tdata_unsafe_free_chunk *partial = os->tdata_unsafe_free_chunk;
if (partial) {
os->tdata_unsafe_free_chunk = NULL;
gc_tdata_unsafe_drain_chunk(partial);
}
}
rbimpl_atomic_size_exchange(&global_objspace->tdata_deferred_free_count, 0,
RBIMPL_ATOMIC_RELAXED);
}
/* Stop the world and run the dfree function for all deferred T_DATAs. */
static void
gc_tdata_unsafe_drain(void)
{
unsigned int lev = RB_GC_VM_LOCK();
if (tdata_deferred_free_count_load() == 0) {
RB_GC_VM_UNLOCK(lev);
return;
}
rb_gc_vm_barrier();
gc_global_snapshot_objspaces();
/* Set during_gc=TRUE and init vm_context for the CURRENT objspace only.
* The no-alloc guard checks only the allocating (=current) objspace's during_gc,
* and rb_gc_get_ec() reads only the current objspace's vm_context.ec. */
rb_objspace_t *objspace = rb_gc_get_objspace();
unsigned int saved_during_gc = gc_during_gc_get(objspace);
dont_gc_on();
rb_gc_initialize_vm_context(&objspace->vm_context);
gc_during_gc_set(objspace, TRUE);
gc_tdata_unsafe_drain_objspaces(global_objspace->global_gc.objspaces,
global_objspace->global_gc.n_objspaces);
gc_during_gc_set(objspace, saved_during_gc);
dont_gc_off();
RB_GC_VM_UNLOCK(lev);
}
static void
gc_tdata_deferred_free_job(void *unused)
{
(void)unused;
size_t count = tdata_deferred_free_count_load();
if (count == 0) return;
if (count < TDATA_DEFERRED_FREE_THRESHOLD && !rb_gc_single_objspace_p()) return;
gc_tdata_unsafe_drain();
}
static inline void
gc_sweep_register_free_slot(rb_objspace_t *objspace, struct heap_page *page, struct gc_sweep_context *ctx, uintptr_t p, short slot_size)
{
rb_asan_unpoison_object(p, false);
((struct RBasic *)p)->flags = 0;
/* Keep a freed slot from carrying its old shareable and shref bits into the next
* object born there; the actual clear happens per bitmap word at the end of
* gc_sweep_page rather than per slot. */
struct free_region *existing_region = ctx->free_region;
if (existing_region) rb_asan_unpoison_object((VALUE)existing_region, false);
if (RB_LIKELY(existing_region && p == existing_region->end)) {
existing_region->end = p + slot_size;
}
else {
struct free_region *free_region = (struct free_region *)p;
free_region->end = p + slot_size;
free_region->next = existing_region;
ctx->free_region = free_region;
}
if (existing_region) rb_asan_poison_object((VALUE)existing_region);
rb_asan_poison_object(p);
}
static inline void
gc_sweep_plane(rb_objspace_t *objspace, rb_heap_t *heap, uintptr_t p, bits_t bitset, struct gc_sweep_context *ctx)
{
struct heap_page *sweep_page = ctx->page;
short slot_size = sweep_page->slot_size;
do {
VALUE vp = (VALUE)p;
GC_ASSERT(vp % sizeof(VALUE) == 0);
rb_asan_unpoison_object(vp, false);
if (bitset & 1) {
switch (BUILTIN_TYPE(vp)) {
case T_MOVED:
if (objspace->flags.during_compacting) {
/* The sweep cursor shouldn't have made it to any
* T_MOVED slots while the compact flag is enabled.
* The sweep cursor and compact cursor move in
* opposite directions, and when they meet references will
* get updated and "during_compacting" should get disabled */
rb_bug("T_MOVED shouldn't be seen until compaction is finished");
}
gc_report(3, objspace, "page_sweep: %s is freed\n", rb_obj_info(vp));
ctx->empty_slots++;
gc_sweep_register_free_slot(objspace, sweep_page, ctx, p, slot_size);
break;
case T_ZOMBIE:
/* already counted */
break;
case T_NONE:
ctx->empty_slots++; /* already freed */
gc_sweep_register_free_slot(objspace, sweep_page, ctx, p, slot_size);
break;
default:
#if RGENGC_CHECK_MODE
/* A local GC must never free a pinned slot; a global GC may (its exact
* mark collects dead shareable objects). Reading the bits here is
* CHECK-only and still valid: the bulk clear runs after the free loop. */
if (ctx->check_pinned_free &&
(MARKED_IN_BITMAP(GET_HEAP_SHAREABLE_BITS(vp), vp) ||
MARKED_IN_BITMAP(GET_HEAP_SHREF_BITS(vp), vp))) {
rb_bug("page_sweep: freeing pinned slot %s (shareable=%d shref=%d single_now=%d)",
rb_obj_info(vp),
(int)!!MARKED_IN_BITMAP(GET_HEAP_SHAREABLE_BITS(vp), vp),
(int)!!MARKED_IN_BITMAP(GET_HEAP_SHREF_BITS(vp), vp),
(int)rb_gc_single_objspace_p());
}
#endif
#if RGENGC_CHECK_MODE
if (!is_full_marking(objspace)) {
if (RVALUE_OLD_P(objspace, vp)) rb_bug("page_sweep: %p - old while minor GC.", (void *)p);
if (RVALUE_REMEMBERED(objspace, vp)) rb_bug("page_sweep: %p - remembered.", (void *)p);
}
#endif
#if RGENGC_CHECK_MODE
#define CHECK(x) if (x(objspace, vp) != FALSE) rb_bug("obj_free: " #x "(%s) != FALSE", rb_obj_info(vp))
CHECK(RVALUE_WB_UNPROTECTED);
CHECK(RVALUE_MARKED);
CHECK(RVALUE_MARKING);
CHECK(RVALUE_UNCOLLECTIBLE);
#undef CHECK
#endif
if (!rb_gc_obj_needs_cleanup_p(vp)) {
(void)VALGRIND_MAKE_MEM_UNDEFINED((void*)p, slot_size);
gc_sweep_register_free_slot(objspace, sweep_page, ctx, p, slot_size);
gc_report(3, objspace, "page_sweep: %s (fast path) is freed\n", rb_obj_info(vp));
ctx->freed_slots++;
}
else {
gc_report(2, objspace, "page_sweep: free %p\n", (void *)p);
if (RB_UNLIKELY(ctx->defer_thread_unsafe_local_sweep && gc_obj_defer_local_free_p(objspace, vp))) {
/* Defer the dfree instead of running it here: it needs the world
* stopped, which a parallel local sweep cannot give it. The slot is reusable
* right away unless we had to create a zombie. */
if (gc_defer_thread_unsafe_free(objspace, vp,
&ctx->trigger_thread_unsafe_sweep_postponed_job)) {
(void)VALGRIND_MAKE_MEM_UNDEFINED((void*)p, slot_size);
gc_sweep_register_free_slot(objspace, sweep_page, ctx, p, slot_size);
ctx->freed_slots++;
}
else {
ctx->final_slots++;
}
break;
}
rb_gc_obj_free_vm_weak_references(vp);
if (gc_obj_free(objspace, vp)) {
(void)VALGRIND_MAKE_MEM_UNDEFINED((void*)p, slot_size);
gc_sweep_register_free_slot(objspace, sweep_page, ctx, p, slot_size);
gc_report(3, objspace, "page_sweep: %s is freed\n", rb_obj_info(vp));
ctx->freed_slots++;
}
else {
ctx->final_slots++;
}
}
break;
}
}
p += slot_size;
bitset >>= 1;
} while (bitset);
}
static inline void
gc_sweep_page(rb_objspace_t *objspace, rb_heap_t *heap, struct gc_sweep_context *ctx)
{
struct heap_page *sweep_page = ctx->page;
GC_ASSERT(sweep_page->heap == heap);
uintptr_t p;
bits_t *bits, bitset;
gc_report(2, objspace, "page_sweep: start.\n");
#if RGENGC_CHECK_MODE
if (!objspace->flags.immediate_sweep) {
GC_ASSERT(sweep_page->flags.before_sweep == TRUE);
}
#endif
sweep_page->flags.before_sweep = FALSE;
sweep_page->free_slots = 0;
asan_unlock_freelist(sweep_page);
sweep_page->free_region = NULL;
asan_lock_freelist(sweep_page);
ctx->free_region = NULL;
p = (uintptr_t)sweep_page->start;
bits = sweep_page->mark_bits;
short slot_size = sweep_page->slot_size;
int total_slots = sweep_page->total_slots;
int bitmap_plane_count = CEILDIV(total_slots, BITS_BITLENGTH);
int out_of_range_bits = total_slots % BITS_BITLENGTH;
if (out_of_range_bits != 0) {
bits[bitmap_plane_count - 1] |= ~(((bits_t)1 << out_of_range_bits) - 1);
}
// Clear wb_unprotected and age bits for all unmarked slots
{
bits_t *wb_unprotected_bits = sweep_page->wb_unprotected_bits;
bits_t *age_bits = sweep_page->age_bits;
for (int i = 0; i < bitmap_plane_count; i++) {
bits_t unmarked = ~bits[i];
wb_unprotected_bits[i] &= ~unmarked;
age_bits[i * 2] &= ~unmarked;
age_bits[i * 2 + 1] &= ~unmarked;
}
}
for (int i = 0; i < bitmap_plane_count; i++) {
bitset = ~bits[i];
if (bitset) {
gc_sweep_plane(objspace, heap, p, bitset, ctx);
}
p += BITS_BITLENGTH * slot_size;
}
/* Bulk-clear the freed slots' shareable and shref bits before the freelist is
* published, so a reused slot is clean. Freed slots are exactly the unmarked ones,
* so `bits &= mark_bits` keeps live shareable objects (which must stay pinned) and
* drops the rest. Pages with neither bit are skipped. */
if (sweep_page->flags.has_shareable_objects || sweep_page->flags.has_shref_objects) {
bits_t *shareable_bits = sweep_page->shareable_bits;
bits_t *shref_bits = sweep_page->shref_bits;
bits_t sh = 0, sr = 0;
for (int i = 0; i < bitmap_plane_count; i++) {
shareable_bits[i] &= bits[i];
shref_bits[i] &= bits[i];
sh |= shareable_bits[i];
sr |= shref_bits[i];
}
if (!sh) sweep_page->flags.has_shareable_objects = FALSE;
if (!sr) sweep_page->flags.has_shref_objects = FALSE;
}
asan_unlock_freelist(sweep_page);
sweep_page->free_region = ctx->free_region;
asan_lock_freelist(sweep_page);
if (!heap->compact_cursor) {
gc_setup_mark_bits(sweep_page);
}
#if GC_PROFILE_MORE_DETAIL
if (gc_prof_enabled(objspace)) {
gc_profile_record *record = gc_prof_record(objspace);
record->removing_objects += ctx->final_slots + ctx->freed_slots;
record->empty_objects += ctx->empty_slots;
}
#endif
if (0) fprintf(stderr, "gc_sweep_page(%"PRIdSIZE"): total_slots: %d, freed_slots: %d, empty_slots: %d, final_slots: %d\n",
rb_gc_count(),
sweep_page->total_slots,
ctx->freed_slots, ctx->empty_slots, ctx->final_slots);
sweep_page->free_slots += ctx->freed_slots + ctx->empty_slots;
sweep_page->heap->total_freed_objects += ctx->freed_slots;
if (heap_pages_deferred_final && !finalizing) {
gc_finalize_deferred_register(objspace);
}
#if RGENGC_CHECK_MODE
int region_slots = 0;
asan_unlock_freelist(sweep_page);
struct free_region *region = sweep_page->free_region;
while (region) {
rb_asan_unpoison_object((VALUE)region, false);
GC_ASSERT(RB_TYPE_P((VALUE)region, T_NONE));
uintptr_t region_start = (uintptr_t)region;
uintptr_t region_end = region->end;
struct free_region *next = region->next;
rb_asan_poison_object((VALUE)region);
GC_ASSERT(region_end > region_start);
GC_ASSERT((region_end - region_start) % slot_size == 0);
region_slots += (int)((region_end - region_start) / slot_size);
region = next;
}
asan_lock_freelist(sweep_page);
if (region_slots != sweep_page->free_slots) {
rb_bug("inconsistent free region slots: expected %d but was %d", sweep_page->free_slots, region_slots);
}
#endif
gc_report(2, objspace, "page_sweep: end.\n");
}
static const char *
gc_mode_name(enum gc_mode mode)
{
switch (mode) {
case gc_mode_none: return "none";
case gc_mode_marking: return "marking";
case gc_mode_sweeping: return "sweeping";
case gc_mode_compacting: return "compacting";
default: rb_bug("gc_mode_name: unknown mode: %d", (int)mode);
}
}
static void
gc_mode_transition(rb_objspace_t *objspace, enum gc_mode mode)
{
#if RGENGC_CHECK_MODE
enum gc_mode prev_mode = gc_mode(objspace);
switch (prev_mode) {
case gc_mode_none:
/* A global GC marks every objspace as one heap (mark_roots on the driver), so an
* individual objspace's mode stays `none` during that mark; the sweep inside the
* barrier then makes the legitimate none -> sweeping transition. */
GC_ASSERT(mode == gc_mode_marking ||
(objspace->flags.during_global_gc && mode == gc_mode_sweeping));
break;
case gc_mode_marking: GC_ASSERT(mode == gc_mode_sweeping); break;
case gc_mode_sweeping: GC_ASSERT(mode == gc_mode_none || mode == gc_mode_compacting); break;
case gc_mode_compacting: GC_ASSERT(mode == gc_mode_none); break;
}
#endif
if (0) fprintf(stderr, "gc_mode_transition: %s->%s\n", gc_mode_name(gc_mode(objspace)), gc_mode_name(mode));
gc_mode_set(objspace, mode);
}
static void
heap_page_flush_alloc_regions(struct heap_page *page, rb_heap_t *heap)
{
struct free_region *chain = heap->newobj.alloc_next_region;
if (heap->newobj.alloc_cursor < heap->newobj.alloc_cursor_end) {
VALUE start = (VALUE)heap->newobj.alloc_cursor;
rb_asan_unpoison_object(start, false);
struct free_region *remnant = (struct free_region *)start;
remnant->flags = 0;
remnant->end = heap->newobj.alloc_cursor_end;
remnant->next = chain;
rb_asan_poison_object(start);
chain = remnant;
}
if (chain) {
asan_unlock_freelist(page);
if (page->free_region) {
struct free_region *p = page->free_region;
rb_asan_unpoison_object((VALUE)p, false);
while (p->next) {
struct free_region *prev = p;
p = p->next;
rb_asan_poison_object((VALUE)prev);
rb_asan_unpoison_object((VALUE)p, false);
}
p->next = chain;
rb_asan_poison_object((VALUE)p);
}
else {
page->free_region = chain;
}
asan_lock_freelist(page);
}
}
static void
gc_sweep_start_heap(rb_objspace_t *objspace, rb_heap_t *heap)
{
heap->sweeping_page = ccan_list_top(&heap->pages, struct heap_page, page_node);
if (heap->sweeping_page) {
objspace->sweeping_heap_count++;
}
heap->free_pages = NULL;
heap->pooled_pages = NULL;
if (!objspace->flags.immediate_sweep) {
struct heap_page *page = NULL;
ccan_list_for_each(&heap->pages, page, page_node) {
page->flags.before_sweep = TRUE;
}
}
}
#if GC_CAN_COMPILE_COMPACTION
static void gc_sort_heap_by_compare_func(rb_objspace_t *objspace, gc_compact_compare_func compare_func);
static int compare_pinned_slots(const void *left, const void *right, void *d);
#endif
/* Return the current allocation page and freelist to their pages, so the sweeper sees a
* consistent heap. */
static void
heap_alloc_state_clear(rb_objspace_t *objspace)
{
objspace->incremental_mark_step_allocated_slots = 0;
for (size_t heap_idx = 0; heap_idx < HEAP_COUNT; heap_idx++) {
rb_heap_t *heap = &heaps[heap_idx];
struct heap_page *page = heap->newobj.alloc_using_page;
RUBY_DEBUG_LOG("heap alloc_using_page:%p cursor:%p", (void *)page, (void *)heap->newobj.alloc_cursor);
if (page) {
heap_page_flush_alloc_regions(page, heap);
}
heap->newobj.alloc_using_page = NULL;
heap->newobj.alloc_cursor = 0;
heap->newobj.alloc_cursor_end = 0;
heap->newobj.alloc_next_region = NULL;
}
}
static void
gc_sweep_freeobj_hooks_page(rb_objspace_t *objspace, struct heap_page *page)
{
bits_t *bits = page->mark_bits;
uintptr_t p = (uintptr_t)page->start;
short slot_size = page->slot_size;
int total_slots = page->total_slots;
int bitmap_plane_count = CEILDIV(total_slots, BITS_BITLENGTH);
int out_of_range_bits = total_slots % BITS_BITLENGTH;
bits_t last_plane_mask = (out_of_range_bits != 0)
? ~(((bits_t)1 << out_of_range_bits) - 1)
: 0;
for (int j = 0; j < bitmap_plane_count; j++) {
bits_t bitset = ~bits[j];
if (j == bitmap_plane_count - 1) {
bitset &= ~last_plane_mask;
}
uintptr_t pp = p;
while (bitset) {
if (bitset & 1) {
VALUE vp = (VALUE)pp;
asan_unpoisoning_object(vp) {
switch (BUILTIN_TYPE(vp)) {
case T_NONE:
case T_ZOMBIE:
case T_MOVED:
break;
default:
rb_gc_event_hook(vp, RUBY_INTERNAL_EVENT_FREEOBJ);
break;
}
}
}
pp += slot_size;
bitset >>= 1;
}
p += BITS_BITLENGTH * slot_size;
}
}
static void
gc_sweep_freeobj_hooks(rb_objspace_t *objspace)
{
for (int i = 0; i < HEAP_COUNT; i++) {
rb_heap_t *heap = &heaps[i];
struct heap_page *page = NULL;
ccan_list_for_each(&heap->pages, page, page_node) {
gc_sweep_freeobj_hooks_page(objspace, page);
}
}
}
static void
gc_sweep_start(rb_objspace_t *objspace)
{
gc_mode_transition(objspace, gc_mode_sweeping);
objspace->rincgc.pooled_slots = 0;
if (RB_UNLIKELY(objspace->hook_events & RUBY_INTERNAL_EVENT_FREEOBJ)) {
/* FREEOBJ is never enabled outside the main objspace
* (rb_objspace_set_event_hook), so this hook, which runs user callbacks,
* cannot fire during a non-main Ractor's lock-free local sweep. */
GC_ASSERT(objspace == global_objspace->main_objspace);
gc_sweep_freeobj_hooks(objspace);
}
#if GC_CAN_COMPILE_COMPACTION
if (objspace->flags.during_compacting) {
gc_sort_heap_by_compare_func(
objspace,
objspace->rcompactor.compare_func ? objspace->rcompactor.compare_func : compare_pinned_slots
);
}
#endif
for (int i = 0; i < HEAP_COUNT; i++) {
rb_heap_t *heap = &heaps[i];
gc_sweep_start_heap(objspace, heap);
/* We should call gc_sweep_finish_heap for size pools with no pages. */
if (heap->sweeping_page == NULL) {
GC_ASSERT(heap->total_pages == 0);
GC_ASSERT(heap->total_slots == 0);
gc_sweep_finish_heap(objspace, heap);
}
}
heap_alloc_state_clear(objspace);
}
static void
gc_sweep_finish_heap(rb_objspace_t *objspace, rb_heap_t *heap)
{
/* Nothing allocates from an ownerless objspace, so it has no growth decision */
if (objspace->flags.ownerless) return;
size_t total_slots = heap->total_slots;
size_t swept_slots = heap->freed_slots + heap->empty_slots;
size_t init_slots = objspace_heap_init_bytes(objspace) / heap->slot_size;
size_t min_free_slots = (size_t)(MAX(total_slots, init_slots) * gc_params.heap_free_slots_min_ratio);
if (swept_slots < min_free_slots &&
/* The heap is a growth heap if it freed more slots than had empty slots. */
((heap->empty_slots == 0 && total_slots > 0) || heap->freed_slots > heap->empty_slots)) {
/* If we don't have enough slots and we have pages on the tomb heap, move
* pages from the tomb heap to the eden heap. This may prevent page
* creation thrashing (frequently allocating and deallocting pages) and
* GC thrashing (running GC more frequently than required). */
struct heap_page *resurrected_page;
while (swept_slots < min_free_slots &&
(resurrected_page = heap_page_resurrect(objspace))) {
heap_add_page(objspace, heap, resurrected_page);
heap_add_freepage(heap, resurrected_page);
swept_slots += resurrected_page->free_slots;
}
if (swept_slots < min_free_slots) {
/* Grow this heap if we are in a major GC or if we haven't run at least
* RVALUE_OLD_AGE minor GC since the last major GC. */
if (is_full_marking(objspace) ||
objspace->profile.count - objspace->rgengc.last_major_gc < RVALUE_OLD_AGE) {
if (objspace->heap_pages.allocatable_bytes < min_free_slots * heap->slot_size) {
heap_allocatable_bytes_expand(objspace, heap, swept_slots, heap->total_slots, heap->slot_size);
}
}
else if (swept_slots < min_free_slots * 7 / 8 &&
objspace->heap_pages.allocatable_bytes < (min_free_slots * 7 / 8 - swept_slots) * heap->slot_size) {
gc_needs_major_flags |= GPR_FLAG_MAJOR_BY_NOFREE;
heap->force_major_gc_count++;
}
}
}
}
#if RGENGC_ESTIMATE_OLDMALLOC
static void
gc_check_oldmalloc(rb_objspace_t *objspace)
{
size_t oldmalloc_increase = gc_oldmalloc_increase_load(objspace);
MALLOC_COUNTERS_LOCK(objspace);
gc_counter_t malloc_at = gc_counter_load_acquire(&objspace->malloc_counters.counters.malloc_at_last_gc);
gc_counter_t free_at = gc_counter_load_acquire(&objspace->malloc_counters.counters.free_at_last_gc);
gc_counter_t malloc_now = gc_counter_load_relaxed(&objspace->malloc_counters.counters.malloc);
gc_counter_t free_now = gc_counter_load_relaxed(&objspace->malloc_counters.counters.free);
gc_counter_t gc_freed = gc_counter_load_relaxed(&objspace->malloc_counters.freed_in_gc);
MALLOC_COUNTERS_UNLOCK(objspace);
gc_counter_t freed = free_now - free_at;
gc_counter_t mutator_freed = freed > gc_freed ? freed - gc_freed : 0;
gc_counter_t allocated = malloc_now - malloc_at;
gc_counter_t unswept = allocated > mutator_freed ? allocated - mutator_freed : 0;
if ((gc_counter_t)oldmalloc_increase <= unswept) return;
oldmalloc_increase -= (size_t)unswept;
if (oldmalloc_increase > objspace->rgengc.oldmalloc_increase_limit) {
gc_needs_major_flags |= GPR_FLAG_MAJOR_BY_OLDMALLOC;
objspace->rgengc.oldmalloc_increase_limit =
(size_t)(objspace->rgengc.oldmalloc_increase_limit * gc_params.oldmalloc_limit_growth_factor);
if (objspace->rgengc.oldmalloc_increase_limit > gc_params.oldmalloc_limit_max) {
objspace->rgengc.oldmalloc_increase_limit = gc_params.oldmalloc_limit_max;
}
}
}
#endif
static void
gc_sweep_finish(rb_objspace_t *objspace)
{
gc_report(1, objspace, "gc_sweep_finish\n");
gc_prof_set_heap_info(objspace);
heap_pages_free_unused_pages(objspace);
if (rb_gc_single_objspace_p() && is_full_marking(objspace)) {
/* gc_marks_finish retains ~2/3 of empty pages in objspace->empty_pages for reuse,
* only the excess reaches the pool. */
page_pool_reclaim(global_objspace);
}
#if RGENGC_ESTIMATE_OLDMALLOC
if (!is_full_marking(objspace)) {
gc_check_oldmalloc(objspace);
}
#endif
for (int i = 0; i < HEAP_COUNT; i++) {
rb_heap_t *heap = &heaps[i];
heap->freed_slots = 0;
heap->empty_slots = 0;
if (!will_be_incremental_marking(objspace)) {
struct heap_page *end_page = heap->free_pages;
if (end_page) {
while (end_page->free_next) end_page = end_page->free_next;
end_page->free_next = heap->pooled_pages;
}
else {
heap->free_pages = heap->pooled_pages;
}
heap->pooled_pages = NULL;
objspace->rincgc.pooled_slots = 0;
}
}
/* Not before: while sweeping is in progress its frees must keep reducing
* malloc_increase (objspace_malloc_increase_body sweeps and retries on it). */
gc_malloc_counters_snapshot_free_at_last_gc(objspace, &objspace->malloc_counters.counters);
/* Leftovers from an earlier multi-Ractor phase: no later sweep can rediscover them
* (their slots are gone), and with one Ractor left the drain's barrier has nothing
* to wait for. */
if (tdata_deferred_free_count_load() > 0 && rb_gc_single_objspace_p()) {
gc_tdata_deferred_free_trigger(objspace);
}
gc_event_hook(objspace, RUBY_INTERNAL_EVENT_GC_END_SWEEP);
gc_mode_transition(objspace, gc_mode_none);
}
static int
gc_sweep_step(rb_objspace_t *objspace, rb_heap_t *heap)
{
struct heap_page *sweep_page = heap->sweeping_page;
int swept_slots = 0;
int pooled_slots = 0;
int sweep_budget = GC_INCREMENTAL_SWEEP_BYTES / heap->slot_size;
int pool_budget = GC_INCREMENTAL_SWEEP_POOL_BYTES / heap->slot_size;
if (sweep_page == NULL) return FALSE;
#if GC_ENABLE_LAZY_SWEEP
gc_prof_sweep_timer_start(objspace);
#endif
/* Per-slot pinned-free assert (gc_sweep_context): check only when this cycle's mark
* ran the pinned walk. The current world state would misfire: a single-world
* cycle leaves dead shareable objects unmarked and its sweep can straddle the switch
* to multi-objspace. A global GC's exact mark does not pin, so it is excluded. */
const bool check_pinned_free = objspace->last_cycle_pinned;
const bool defer_thread_unsafe_local_sweep =
!rb_gc_single_objspace_p() && !objspace->flags.during_global_gc;
bool trigger_thread_unsafe_sweep_postponed_job = false;
do {
RUBY_DEBUG_LOG("sweep_page:%p", (void *)sweep_page);
struct gc_sweep_context ctx = {
.page = sweep_page,
.final_slots = 0,
.freed_slots = 0,
.empty_slots = 0,
.check_pinned_free = check_pinned_free,
.defer_thread_unsafe_local_sweep = defer_thread_unsafe_local_sweep,
.trigger_thread_unsafe_sweep_postponed_job = trigger_thread_unsafe_sweep_postponed_job,
};
gc_sweep_page(objspace, heap, &ctx);
int free_slots = ctx.freed_slots + ctx.empty_slots;
trigger_thread_unsafe_sweep_postponed_job = ctx.trigger_thread_unsafe_sweep_postponed_job;
RUBY_DTRACE_GC_HOOK(SWEEP_PAGE, ctx.page->slot_size, ctx.final_slots, ctx.freed_slots, ctx.empty_slots);
heap->sweeping_page = ccan_list_next(&heap->pages, sweep_page, page_node);
if (free_slots == sweep_page->total_slots &&
(heap->total_pages > 1 || objspace->flags.ownerless)) {
/* There are no living objects, so move this page to the global empty pages.
* The last one stays so the heap has something to grow from -- except in an
* ownerless objspace, which will never allocate again. */
heap_page_detach_empty(objspace, heap, sweep_page);
}
else if (free_slots > 0) {
heap->freed_slots += ctx.freed_slots;
heap->empty_slots += ctx.empty_slots;
if (pooled_slots < pool_budget) {
heap_add_poolpage(objspace, heap, sweep_page);
pooled_slots += free_slots;
}
else {
heap_add_freepage(heap, sweep_page);
swept_slots += free_slots;
if (swept_slots > sweep_budget) {
break;
}
}
}
else {
sweep_page->free_next = NULL;
}
} while ((sweep_page = heap->sweeping_page));
if (trigger_thread_unsafe_sweep_postponed_job) {
gc_report(2, objspace, "thread-unsafe sweep postponed job triggered\n");
gc_tdata_deferred_free_trigger(objspace);
}
if (!heap->sweeping_page) {
objspace->sweeping_heap_count--;
GC_ASSERT(objspace->sweeping_heap_count >= 0);
gc_sweep_finish_heap(objspace, heap);
if (!has_sweeping_pages(objspace)) {
gc_sweep_finish(objspace);
}
}
#if GC_ENABLE_LAZY_SWEEP
gc_prof_sweep_timer_stop(objspace);
#endif
return heap->free_pages != NULL;
}
static void
gc_sweep_rest(rb_objspace_t *objspace)
{
for (int i = 0; i < HEAP_COUNT; i++) {
rb_heap_t *heap = &heaps[i];
while (heap->sweeping_page) {
gc_sweep_step(objspace, heap);
}
}
/* An objspace with no live pages never runs gc_sweep_step and so never reaches
* gc_sweep_finish, leaving mode at sweeping or compacting until the next cycle's
* gc_sweep_start asserts. If every heap is swept out, settle it to none here. */
if (gc_mode(objspace) != gc_mode_none && !has_sweeping_pages(objspace)) {
gc_sweep_finish(objspace);
}
}
static void
gc_sweep_continue(rb_objspace_t *objspace, rb_heap_t *sweep_heap)
{
GC_ASSERT(dont_gc_val() == FALSE || objspace->profile.latest_gc_info & GPR_FLAG_METHOD);
if (!GC_ENABLE_LAZY_SWEEP) return;
gc_sweeping_enter(objspace);
for (int i = 0; i < HEAP_COUNT; i++) {
rb_heap_t *heap = &heaps[i];
if (gc_sweep_step(objspace, heap)) {
GC_ASSERT(heap->free_pages != NULL);
}
else if (heap == sweep_heap) {
if (objspace->empty_pages_count > 0 || objspace->heap_pages.allocatable_bytes > 0) {
/* [Bug #21548]
*
* If this heap is the heap we want to sweep, but we weren't able
* to free any slots, but we also either have empty pages or could
* allocate new pages, then we want to preemptively claim a page
* because it's possible that sweeping another heap will call
* gc_sweep_finish_heap, which may use up all of the
* empty/allocatable pages. If other heaps are not finished sweeping
* then we do not finish this GC and we will end up triggering a new
* GC cycle during this GC phase. */
heap_page_allocate_and_initialize(objspace, heap);
GC_ASSERT(heap->free_pages != NULL);
}
else {
/* Not allowed to create a new page so finish sweeping. */
gc_sweep_rest(objspace);
GC_ASSERT(gc_mode(objspace) == gc_mode_none);
break;
}
}
}
gc_sweeping_exit(objspace);
}
static void
gc_sweep_step_for_malloc(rb_objspace_t *objspace)
{
GC_ASSERT(is_lazy_sweeping(objspace));
unsigned int lock_lev;
gc_enter(objspace, gc_enter_event_continue, &lock_lev);
gc_sweeping_enter(objspace);
for (int i = 0; i < HEAP_COUNT; i++) {
rb_heap_t *heap = &heaps[i];
gc_sweep_step(objspace, heap);
}
gc_sweeping_exit(objspace);
gc_exit(objspace, gc_enter_event_continue, &lock_lev);
}
static bool gc_global_pointer_to_heap_p(const void *ptr);
VALUE
rb_gc_impl_location(void *objspace_ptr, VALUE value)
{
rb_objspace_t *objspace = objspace_ptr;
VALUE destination;
/* A local (single-objspace) compaction never moves another objspace's objects, so
* leave foreign references alone. A compacting global GC moves objects everywhere
* under the barrier, so there every objspace's heap is searched for forwarding. */
if (RB_UNLIKELY(objspace->flags.during_global_gc)
? !gc_global_pointer_to_heap_p((void *)value)
: !is_pointer_to_heap(objspace_ptr, (void *)value)) {
return value;
}
asan_unpoisoning_object(value) {
if (BUILTIN_TYPE(value) == T_MOVED) {
destination = (VALUE)RMOVED(value)->destination;
GC_ASSERT(BUILTIN_TYPE(destination) != T_NONE);
}
else {
destination = value;
}
}
return destination;
}
#if GC_CAN_COMPILE_COMPACTION
static void
invalidate_moved_plane(rb_objspace_t *objspace, struct heap_page *page, uintptr_t p, bits_t bitset)
{
if (bitset) {
do {
if (bitset & 1) {
VALUE forwarding_object = (VALUE)p;
VALUE object;
if (BUILTIN_TYPE(forwarding_object) == T_MOVED) {
GC_ASSERT(RVALUE_PINNED(objspace, forwarding_object));
GC_ASSERT(!RVALUE_MARKED(objspace, forwarding_object));
CLEAR_IN_BITMAP(GET_HEAP_PINNED_BITS(forwarding_object), forwarding_object);
object = rb_gc_impl_location(objspace, forwarding_object);
gc_move(objspace, object, forwarding_object, GET_HEAP_PAGE(object), page);
/* forwarding_object is now our actual object, and "object"
* is the free slot for the original page */
struct heap_page *orig_page = GET_HEAP_PAGE(object);
orig_page->free_slots++;
RVALUE_AGE_SET_BITMAP(object, 0);
heap_page_add_free_region(objspace, orig_page, object);
GC_ASSERT(RVALUE_MARKED(objspace, forwarding_object));
GC_ASSERT(BUILTIN_TYPE(forwarding_object) != T_MOVED);
GC_ASSERT(BUILTIN_TYPE(forwarding_object) != T_NONE);
}
}
p += page->slot_size;
bitset >>= 1;
} while (bitset);
}
}
static void
invalidate_moved_page(rb_objspace_t *objspace, struct heap_page *page)
{
int i;
bits_t *mark_bits, *pin_bits;
bits_t bitset;
short slot_size = page->slot_size;
int total_slots = page->total_slots;
int bitmap_plane_count = CEILDIV(total_slots, BITS_BITLENGTH);
mark_bits = page->mark_bits;
pin_bits = page->pinned_bits;
uintptr_t p = page->start;
for (i=0; i < bitmap_plane_count; i++) {
/* Moved objects are pinned but never marked. We reuse the pin bits
* to indicate there is a moved object in this slot. */
bitset = pin_bits[i] & ~mark_bits[i];
invalidate_moved_plane(objspace, page, p, bitset);
p += BITS_BITLENGTH * slot_size;
}
}
#endif
static void
gc_compact_start(rb_objspace_t *objspace)
{
struct heap_page *page = NULL;
gc_mode_transition(objspace, gc_mode_compacting);
for (int i = 0; i < HEAP_COUNT; i++) {
rb_heap_t *heap = &heaps[i];
ccan_list_for_each(&heap->pages, page, page_node) {
page->flags.before_sweep = TRUE;
}
heap->compact_cursor = ccan_list_tail(&heap->pages, struct heap_page, page_node);
heap->compact_cursor_index = 0;
}
if (gc_prof_enabled(objspace)) {
gc_profile_record *record = gc_prof_record(objspace);
record->moved_objects = objspace->rcompactor.total_moved;
}
memset(objspace->rcompactor.considered_count_table, 0, T_MASK * sizeof(size_t));
memset(objspace->rcompactor.moved_count_table, 0, T_MASK * sizeof(size_t));
memset(objspace->rcompactor.moved_up_count_table, 0, T_MASK * sizeof(size_t));
memset(objspace->rcompactor.moved_down_count_table, 0, T_MASK * sizeof(size_t));
/* Set up read barrier for pages containing MOVED objects */
/* A compacting global GC installs the read barrier once for every objspace. */
if (!global_objspace->global_gc.compacting) install_handlers();
}
static void gc_sweep_compact(rb_objspace_t *objspace);
static void
gc_sweep(rb_objspace_t *objspace)
{
gc_sweeping_enter(objspace);
const unsigned int immediate_sweep = objspace->flags.immediate_sweep;
gc_report(1, objspace, "gc_sweep: immediate: %d\n", immediate_sweep);
gc_sweep_start(objspace);
if (objspace->flags.during_compacting) {
rb_hrtime_t compact_start_time = gc_prof_enabled(objspace) ? rb_hrtime_now() : 0;
gc_sweep_compact(objspace);
if (gc_prof_enabled(objspace)) {
rb_hrtime_t compact_wall_time = elapsed_hrtime_from(compact_start_time);
gc_profile_record *record = gc_prof_record(objspace);
record->gc_compact_wall_time = rb_hrtime_add(record->gc_compact_wall_time,
compact_wall_time);
objspace->profile.gc_sweep_excluded_wall_time = rb_hrtime_add(
objspace->profile.gc_sweep_excluded_wall_time,
compact_wall_time);
}
}
if (immediate_sweep) {
#if !GC_ENABLE_LAZY_SWEEP
gc_prof_sweep_timer_start(objspace);
#endif
gc_sweep_rest(objspace);
#if !GC_ENABLE_LAZY_SWEEP
gc_prof_sweep_timer_stop(objspace);
#endif
}
else {
/* Sweep every size pool. */
for (int i = 0; i < HEAP_COUNT; i++) {
rb_heap_t *heap = &heaps[i];
gc_sweep_step(objspace, heap);
}
}
gc_sweeping_exit(objspace);
}
/* Marking - Marking stack */
static stack_chunk_t *
stack_chunk_alloc(void)
{
stack_chunk_t *res;
res = malloc(sizeof(stack_chunk_t));
if (!res)
rb_memerror();
return res;
}
static inline int
is_mark_stack_empty(mark_stack_t *stack)
{
return stack->chunk == NULL;
}
static size_t
mark_stack_size(mark_stack_t *stack)
{
size_t size = stack->index;
stack_chunk_t *chunk = stack->chunk ? stack->chunk->next : NULL;
while (chunk) {
size += stack->limit;
chunk = chunk->next;
}
return size;
}
static void
add_stack_chunk_cache(mark_stack_t *stack, stack_chunk_t *chunk)
{
chunk->next = stack->cache;
stack->cache = chunk;
stack->cache_size++;
}
static void
shrink_stack_chunk_cache(mark_stack_t *stack)
{
stack_chunk_t *chunk;
if (stack->unused_cache_size > (stack->cache_size/2)) {
chunk = stack->cache;
stack->cache = stack->cache->next;
stack->cache_size--;
free(chunk);
}
stack->unused_cache_size = stack->cache_size;
}
static void
push_mark_stack_chunk(mark_stack_t *stack)
{
stack_chunk_t *next;
GC_ASSERT(stack->index == stack->limit);
if (stack->cache_size > 0) {
next = stack->cache;
stack->cache = stack->cache->next;
stack->cache_size--;
if (stack->unused_cache_size > stack->cache_size)
stack->unused_cache_size = stack->cache_size;
}
else {
next = stack_chunk_alloc();
}
next->next = stack->chunk;
stack->chunk = next;
stack->index = 0;
}
static void
pop_mark_stack_chunk(mark_stack_t *stack)
{
stack_chunk_t *prev;
prev = stack->chunk->next;
GC_ASSERT(stack->index == 0);
add_stack_chunk_cache(stack, stack->chunk);
stack->chunk = prev;
stack->index = stack->limit;
}
static void
mark_stack_chunk_list_free(stack_chunk_t *chunk)
{
stack_chunk_t *next = NULL;
while (chunk != NULL) {
next = chunk->next;
free(chunk);
chunk = next;
}
}
static void
free_stack_chunks(mark_stack_t *stack)
{
mark_stack_chunk_list_free(stack->chunk);
}
static void
mark_stack_free_cache(mark_stack_t *stack)
{
mark_stack_chunk_list_free(stack->cache);
stack->cache_size = 0;
stack->unused_cache_size = 0;
}
static void
push_mark_stack(mark_stack_t *stack, VALUE obj)
{
switch (BUILTIN_TYPE(obj)) {
case T_OBJECT:
case T_CLASS:
case T_MODULE:
case T_FLOAT:
case T_STRING:
case T_REGEXP:
case T_ARRAY:
case T_HASH:
case T_STRUCT:
case T_BIGNUM:
case T_FILE:
case T_DATA:
case T_MATCH:
case T_COMPLEX:
case T_RATIONAL:
case T_TRUE:
case T_FALSE:
case T_SYMBOL:
case T_IMEMO:
case T_ICLASS:
if (stack->index == stack->limit) {
push_mark_stack_chunk(stack);
}
stack->chunk->data[stack->index++] = obj;
return;
case T_NONE:
case T_NIL:
case T_FIXNUM:
case T_MOVED:
case T_ZOMBIE:
case T_UNDEF:
case T_MASK:
rb_bug("push_mark_stack() called for broken object");
break;
case T_NODE:
rb_bug("push_mark_stack: unexpected T_NODE object");
break;
}
rb_bug("rb_gc_mark(): unknown data type 0x%x(%p) %s",
BUILTIN_TYPE(obj), (void *)obj,
is_pointer_to_heap((rb_objspace_t *)rb_gc_get_objspace(), (void *)obj) ? "corrupted object" : "non object");
}
static int
pop_mark_stack(mark_stack_t *stack, VALUE *data)
{
if (is_mark_stack_empty(stack)) {
return FALSE;
}
if (stack->index == 1) {
*data = stack->chunk->data[--stack->index];
pop_mark_stack_chunk(stack);
}
else {
*data = stack->chunk->data[--stack->index];
}
return TRUE;
}
static void
init_mark_stack(mark_stack_t *stack)
{
int i;
MEMZERO(stack, mark_stack_t, 1);
stack->index = stack->limit = STACK_CHUNK_SIZE;
for (i=0; i < 4; i++) {
add_stack_chunk_cache(stack, stack_chunk_alloc());
}
stack->unused_cache_size = stack->cache_size;
}
/* Marking */
ALWAYS_INLINE(static int gc_mark_set(rb_objspace_t *objspace, VALUE obj));
ALWAYS_INLINE(static void gc_mark_check_t_none(rb_objspace_t *objspace, VALUE obj));
ALWAYS_INLINE(static void rgengc_check_relation(rb_objspace_t *objspace, VALUE obj));
ALWAYS_INLINE(static void gc_aging(rb_objspace_t *objspace, VALUE obj));
ALWAYS_INLINE(static void gc_grey(rb_objspace_t *objspace, VALUE obj));
static void
rgengc_check_relation(rb_objspace_t *objspace, VALUE obj)
{
if (objspace->rgengc.parent_object_old_p) {
if (!RVALUE_OLD_P(objspace, obj)) {
rgengc_remember(objspace, objspace->rgengc.parent_object);
/* It is in the rememberset now, so its remaining children have nothing left
* to ask for: stop testing them. */
objspace->rgengc.parent_object_old_p = false;
}
}
}
static inline int
gc_mark_set(rb_objspace_t *objspace, VALUE obj)
{
if (RVALUE_MARKED(objspace, obj)) return 0;
MARK_IN_BITMAP(GET_HEAP_MARK_BITS(obj), obj);
return 1;
}
static void
gc_aging(rb_objspace_t *objspace, VALUE obj)
{
/* Disable aging if Major GC's are disabled. This will prevent longish lived
* objects filling up the heap at the expense of marking many more objects.
*
* We should always pre-warm our process when disabling majors, by running
* GC manually several times so that most objects likely to become oldgen
* are already oldgen.
*/
if(!gc_config_full_mark_val)
return;
struct heap_page *page = GET_HEAP_PAGE(obj);
GC_ASSERT(RVALUE_MARKING(objspace, obj) == FALSE);
check_rvalue_consistency(objspace, obj);
if (!RVALUE_PAGE_WB_UNPROTECTED(page, obj)) {
if (!RVALUE_OLD_P(objspace, obj)) {
int t = BUILTIN_TYPE(obj);
if (t == T_CLASS || t == T_MODULE || t == T_ICLASS) {
gc_report(3, objspace, "gc_aging: YOUNG class: %s\n", rb_obj_info(obj));
RVALUE_AGE_SET(obj, RVALUE_OLD_AGE);
RVALUE_OLD_UNCOLLECTIBLE_SET(objspace, obj);
}
else {
gc_report(3, objspace, "gc_aging: YOUNG: %s\n", rb_obj_info(obj));
RVALUE_AGE_INC(objspace, obj);
}
}
else if (is_full_marking(objspace)) {
GC_ASSERT(RVALUE_PAGE_UNCOLLECTIBLE(page, obj) == FALSE);
RVALUE_PAGE_OLD_UNCOLLECTIBLE_SET(objspace, page, obj);
}
}
check_rvalue_consistency(objspace, obj);
page->objspace->marked_slots++;
}
static void
gc_grey(rb_objspace_t *objspace, VALUE obj)
{
#if RGENGC_CHECK_MODE
if (RVALUE_MARKED(objspace, obj) == FALSE) rb_bug("gc_grey: %s is not marked.", rb_obj_info(obj));
if (RVALUE_MARKING(objspace, obj) == TRUE) rb_bug("gc_grey: %s is marking/remembered.", rb_obj_info(obj));
#endif
if (is_incremental_marking(objspace)) {
MARK_IN_BITMAP(GET_HEAP_MARKING_BITS(obj), obj);
}
if (RB_FL_TEST_RAW(obj, RUBY_FL_WEAK_REFERENCE)) {
rb_darray_append_without_gc(&objspace->weak_references, obj);
}
push_mark_stack(&objspace->mark_stack, obj);
}
static inline void
gc_mark_check_t_none(rb_objspace_t *objspace, VALUE obj)
{
if (RB_UNLIKELY(BUILTIN_TYPE(obj) == T_NONE)) {
enum {info_size = 256};
char obj_info_buf[info_size];
rb_raw_obj_info(obj_info_buf, info_size, obj);
char parent_obj_info_buf[info_size];
rb_raw_obj_info(parent_obj_info_buf, info_size, objspace->rgengc.parent_object);
rb_bug("try to mark T_NONE object (obj: %s, parent: %s)", obj_info_buf, parent_obj_info_buf);
}
}
static void
gc_mark(rb_objspace_t *objspace, VALUE obj)
{
GC_ASSERT(during_gc);
GC_ASSERT(!objspace->flags.during_reference_updating);
/* Never step into another objspace: a foreign object is a live leaf whose liveness
* belongs to its owner, so touching its bitmaps here would be unsound. A global GC
* lifts this: everyone is stopped and the bits live on the object's own page. */
if (gc_skip_foreign_object_p(objspace, obj)) {
return;
}
if (RB_UNLIKELY(objspace->flags.during_global_gc)) {
/* Recompute the shref of every shareable -> unshareable edge, within and across
* objspaces: the clear pass dropped all shref bits and the write barrier
* maintains them from here on. */
VALUE parent = objspace->rgengc.parent_object;
if (!UNDEF_P(parent) && parent != Qfalse &&
RB_FL_TEST_RAW(parent, RUBY_FL_SHAREABLE) &&
!RB_FL_TEST_RAW(obj, RUBY_FL_SHAREABLE)) {
struct heap_page *page = GET_HEAP_PAGE(obj);
_MARK_IN_BITMAP(page->shref_bits, page, obj);
page->flags.has_shref_objects = TRUE;
}
}
rgengc_check_relation(objspace, obj);
if (!gc_mark_set(objspace, obj)) return; /* already marked */
if (0) { // for debug GC marking miss
RUBY_DEBUG_LOG("%p (%s) parent:%p (%s)",
(void *)obj, obj_type_name(obj),
(void *)objspace->rgengc.parent_object, obj_type_name(objspace->rgengc.parent_object));
}
gc_mark_check_t_none(objspace, obj);
gc_aging(objspace, obj);
gc_grey(objspace, obj);
}
static inline void
gc_pin(rb_objspace_t *objspace, VALUE obj)
{
GC_ASSERT(!SPECIAL_CONST_P(obj));
if (RB_UNLIKELY(objspace->flags.during_compacting)) {
/* Never write a foreign page's pinned bit (a global GC may: everyone is stopped). */
if (gc_skip_foreign_object_p(objspace, obj)) return;
if (RB_LIKELY(during_gc)) {
if (!RVALUE_PINNED(objspace, obj)) {
GC_ASSERT(GET_HEAP_PAGE(obj)->pinned_slots <= GET_HEAP_PAGE(obj)->total_slots);
GET_HEAP_PAGE(obj)->pinned_slots++;
MARK_IN_BITMAP(GET_HEAP_PINNED_BITS(obj), obj);
}
}
}
}
static inline void
gc_mark_and_pin(rb_objspace_t *objspace, VALUE obj)
{
gc_pin(objspace, obj);
gc_mark(objspace, obj);
}
void
rb_gc_impl_mark_and_move(void *objspace_ptr, VALUE *ptr)
{
rb_objspace_t *objspace = objspace_ptr;
if (RB_UNLIKELY(objspace->flags.during_reference_updating)) {
GC_ASSERT(objspace->flags.during_compacting);
GC_ASSERT(during_gc);
VALUE destination = rb_gc_impl_location(objspace, *ptr);
if (destination != *ptr) {
*ptr = destination;
}
}
else {
gc_mark(objspace, *ptr);
}
}
void
rb_gc_impl_mark(void *objspace_ptr, VALUE obj)
{
rb_objspace_t *objspace = objspace_ptr;
gc_mark(objspace, obj);
}
void
rb_gc_impl_mark_and_pin(void *objspace_ptr, VALUE obj)
{
rb_objspace_t *objspace = objspace_ptr;
gc_mark_and_pin(objspace, obj);
}
/* A word scanned conservatively by a global GC can point into any objspace, so ownership
* is decided against the driver's snapshot of every objspace (the bits then land on the
* owner's page through gc_mark and gc_pin). */
static bool
gc_global_pointer_to_heap_p(const void *ptr)
{
const rb_global_objspace_t *g = global_objspace;
uintptr_t p = (uintptr_t)ptr;
if (p < g->page_index.lomem || p > g->page_index.himem) return false;
if (p % sizeof(VALUE) != 0) return false;
struct heap_page **res = bsearch(ptr, g->page_index.pages, g->page_index.n_pages,
sizeof(struct heap_page *), ptr_in_page_body_p);
if (res == NULL) return false;
struct heap_page *page = *res;
if (heap_page_in_global_empty_pages_pool(page->objspace, page)) return false;
if (p < page->start) return false;
if (p >= page->start + (page->total_slots * page->slot_size)) return false;
if ((p - page->start) % page->slot_size != 0) return false;
return true;
}
void
rb_gc_impl_mark_maybe(void *objspace_ptr, VALUE obj)
{
rb_objspace_t *objspace = objspace_ptr;
(void)VALGRIND_MAKE_MEM_DEFINED(&obj, sizeof(obj));
if (RB_UNLIKELY(objspace->flags.during_global_gc)
? gc_global_pointer_to_heap_p((void *)obj)
: is_pointer_to_heap(objspace, (void *)obj)) {
asan_unpoisoning_object(obj) {
/* Garbage can live on the stack, so do not mark or pin */
switch (BUILTIN_TYPE(obj)) {
case T_ZOMBIE:
case T_NONE:
break;
default:
gc_mark_and_pin(objspace, obj);
break;
}
}
}
}
static int
pin_value(st_data_t key, st_data_t value, st_data_t data)
{
rb_gc_impl_mark_and_pin((void *)data, (VALUE)value);
return ST_CONTINUE;
}
static inline void
gc_mark_set_parent_raw(rb_objspace_t *objspace, VALUE obj, bool old_p)
{
asan_unpoison_memory_region(&objspace->rgengc.parent_object, sizeof(objspace->rgengc.parent_object), false);
asan_unpoison_memory_region(&objspace->rgengc.parent_object_old_p, sizeof(objspace->rgengc.parent_object_old_p), false);
objspace->rgengc.parent_object = obj;
objspace->rgengc.parent_object_old_p = old_p;
}
static inline void
gc_mark_set_parent(rb_objspace_t *objspace, VALUE obj)
{
gc_mark_set_parent_raw(objspace, obj, RVALUE_OLD_P(objspace, obj));
}
static inline void
gc_mark_set_parent_invalid(rb_objspace_t *objspace)
{
asan_poison_memory_region(&objspace->rgengc.parent_object, sizeof(objspace->rgengc.parent_object));
asan_poison_memory_region(&objspace->rgengc.parent_object_old_p, sizeof(objspace->rgengc.parent_object_old_p));
}
static void pinned_roots_mark(rb_objspace_t *objspace, rb_heap_t *heap);
static void
mark_roots(rb_objspace_t *objspace, const char **categoryp)
{
VALUE objspace_guard = (VALUE)objspace;
#define MARK_CHECKPOINT(category) do { \
if (categoryp) *categoryp = category; \
} while (0)
/* Pinning shareable objects and shrefs runs at the end of marking (gc_marks_finish),
* not here: after the full walk it only has to touch what ordinary marking missed,
* which is both cheap and a useful retention metric. */
MARK_CHECKPOINT("objspace");
gc_mark_set_parent_raw(objspace, Qundef, false);
if (objspace->flags.during_global_gc) {
/* Pin the finalizer tables of every objspace, zombies included.
* (finalizer_table is a macro over the local "objspace".) */
rb_objspace_t *const driver = objspace;
for (size_t i = 0; i < global_objspace->global_gc.n_objspaces; i++) {
rb_objspace_t *objspace = global_objspace->global_gc.objspaces[i];
if (finalizer_table != NULL) {
st_foreach(finalizer_table, pin_value, (st_data_t)driver);
}
}
}
else if (finalizer_table != NULL) {
st_foreach(finalizer_table, pin_value, (st_data_t)objspace);
}
if (stress_to_class) rb_gc_mark(stress_to_class);
rb_gc_save_machine_context();
rb_gc_mark_roots(objspace, categoryp);
/* Keep this frame, including its saved registers, until root marking has
* scanned the machine stack. */
RB_GC_GUARD(objspace_guard);
gc_mark_set_parent_invalid(objspace);
}
static void
gc_mark_children(rb_objspace_t *objspace, VALUE obj)
{
gc_mark_set_parent(objspace, obj);
rb_gc_mark_children(objspace, obj);
gc_mark_set_parent_invalid(objspace);
}
/**
* incremental: 0 -> not incremental (do all)
* incremental: n -> mark at most `n' objects
*/
static inline int
gc_mark_stacked_objects(rb_objspace_t *objspace, int incremental, size_t count)
{
mark_stack_t *mstack = &objspace->mark_stack;
VALUE obj;
size_t marked_slots_at_the_beginning = objspace->marked_slots;
size_t popped_count = 0;
while (pop_mark_stack(mstack, &obj)) {
if (obj == Qundef) continue; /* skip */
if (RGENGC_CHECK_MODE && !RVALUE_MARKED(objspace, obj)) {
rb_bug("gc_mark_stacked_objects: %s is not marked.", rb_obj_info(obj));
}
gc_mark_children(objspace, obj);
popped_count++;
if (incremental) {
if (RGENGC_CHECK_MODE && !RVALUE_MARKING(objspace, obj)) {
rb_bug("gc_mark_stacked_objects: incremental, but marking bit is 0");
}
CLEAR_IN_BITMAP(GET_HEAP_MARKING_BITS(obj), obj);
if (popped_count + (objspace->marked_slots - marked_slots_at_the_beginning) > count) {
break;
}
}
else {
/* just ignore marking bits */
}
}
RUBY_DTRACE_GC_HOOK(MARK_STACKED_OBJECTS, popped_count);
if (RGENGC_CHECK_MODE >= 3) gc_verify_internal_consistency(objspace);
if (is_mark_stack_empty(mstack)) {
shrink_stack_chunk_cache(mstack);
return TRUE;
}
else {
return FALSE;
}
}
static int
gc_mark_stacked_objects_incremental(rb_objspace_t *objspace, size_t count)
{
return gc_mark_stacked_objects(objspace, TRUE, count);
}
static int
gc_mark_stacked_objects_all(rb_objspace_t *objspace)
{
return gc_mark_stacked_objects(objspace, FALSE, 0);
}
#if RGENGC_CHECK_MODE >= 4
#define MAKE_ROOTSIG(obj) (((VALUE)(obj) << 1) | 0x01)
#define IS_ROOTSIG(obj) ((VALUE)(obj) & 0x01)
#define GET_ROOTSIG(obj) ((const char *)((VALUE)(obj) >> 1))
struct reflist {
VALUE *list;
int pos;
int size;
};
static struct reflist *
reflist_create(VALUE obj)
{
struct reflist *refs = xmalloc(sizeof(struct reflist));
refs->size = 1;
refs->list = ALLOC_N(VALUE, refs->size);
refs->list[0] = obj;
refs->pos = 1;
return refs;
}
static void
reflist_destruct(struct reflist *refs)
{
xfree(refs->list);
xfree(refs);
}
static void
reflist_add(struct reflist *refs, VALUE obj)
{
if (refs->pos == refs->size) {
refs->size *= 2;
SIZED_REALLOC_N(refs->list, VALUE, refs->size, refs->size/2);
}
refs->list[refs->pos++] = obj;
}
static void
reflist_dump(struct reflist *refs)
{
int i;
for (i=0; i<refs->pos; i++) {
VALUE obj = refs->list[i];
if (IS_ROOTSIG(obj)) { /* root */
fprintf(stderr, "<root@%s>", GET_ROOTSIG(obj));
}
else {
fprintf(stderr, "<%s>", rb_obj_info(obj));
}
if (i+1 < refs->pos) fprintf(stderr, ", ");
}
}
static int
reflist_referred_from_machine_context(struct reflist *refs)
{
int i;
for (i=0; i<refs->pos; i++) {
VALUE obj = refs->list[i];
if (IS_ROOTSIG(obj) && strcmp(GET_ROOTSIG(obj), "machine_context") == 0) return 1;
}
return 0;
}
struct allrefs {
rb_objspace_t *objspace;
/* a -> obj1
* b -> obj1
* c -> obj1
* c -> obj2
* d -> obj3
* #=> {obj1 => [a, b, c], obj2 => [c, d]}
*/
struct st_table *references;
const char *category;
VALUE root_obj;
mark_stack_t mark_stack;
};
static int
allrefs_add(struct allrefs *data, VALUE obj)
{
struct reflist *refs;
st_data_t r;
if (st_lookup(data->references, obj, &r)) {
refs = (struct reflist *)r;
reflist_add(refs, data->root_obj);
return 0;
}
else {
refs = reflist_create(data->root_obj);
st_insert(data->references, obj, (st_data_t)refs);
return 1;
}
}
static void
allrefs_i(VALUE obj, void *ptr)
{
struct allrefs *data = (struct allrefs *)ptr;
if (allrefs_add(data, obj)) {
push_mark_stack(&data->mark_stack, obj);
}
}
static void
allrefs_roots_i(VALUE obj, void *ptr)
{
struct allrefs *data = (struct allrefs *)ptr;
if (strlen(data->category) == 0) rb_bug("!!!");
data->root_obj = MAKE_ROOTSIG(data->category);
if (allrefs_add(data, obj)) {
push_mark_stack(&data->mark_stack, obj);
}
}
#define PUSH_MARK_FUNC_DATA(v) do { \
struct gc_mark_func_data_struct *prev_mark_func_data = GET_VM()->gc.mark_func_data; \
GET_VM()->gc.mark_func_data = (v);
#define POP_MARK_FUNC_DATA() GET_VM()->gc.mark_func_data = prev_mark_func_data;} while (0)
static st_table *
objspace_allrefs(rb_objspace_t *objspace)
{
struct allrefs data;
struct gc_mark_func_data_struct mfd;
VALUE obj;
int prev_dont_gc = dont_gc_val();
dont_gc_on();
data.objspace = objspace;
data.references = st_init_numtable();
init_mark_stack(&data.mark_stack);
mfd.mark_func = allrefs_roots_i;
mfd.data = &data;
/* traverse root objects */
PUSH_MARK_FUNC_DATA(&mfd);
GET_VM()->gc.mark_func_data = &mfd;
mark_roots(objspace, &data.category);
POP_MARK_FUNC_DATA();
/* traverse rest objects reachable from root objects */
while (pop_mark_stack(&data.mark_stack, &obj)) {
rb_objspace_reachable_objects_from_unlocked(data.root_obj = obj, allrefs_i, &data);
}
free_stack_chunks(&data.mark_stack);
dont_gc_set(prev_dont_gc);
return data.references;
}
static int
objspace_allrefs_destruct_i(st_data_t key, st_data_t value, st_data_t ptr)
{
struct reflist *refs = (struct reflist *)value;
reflist_destruct(refs);
return ST_CONTINUE;
}
static void
objspace_allrefs_destruct(struct st_table *refs)
{
st_foreach(refs, objspace_allrefs_destruct_i, 0);
st_free_table(refs);
}
#if RGENGC_CHECK_MODE >= 5
static int
allrefs_dump_i(st_data_t k, st_data_t v, st_data_t ptr)
{
VALUE obj = (VALUE)k;
struct reflist *refs = (struct reflist *)v;
fprintf(stderr, "[allrefs_dump_i] %s <- ", rb_obj_info(obj));
reflist_dump(refs);
fprintf(stderr, "\n");
return ST_CONTINUE;
}
static void
allrefs_dump(rb_objspace_t *objspace)
{
VALUE size = objspace->rgengc.allrefs_table->num_entries;
fprintf(stderr, "[all refs] (size: %"PRIuVALUE")\n", size);
st_foreach(objspace->rgengc.allrefs_table, allrefs_dump_i, 0);
}
#endif
static int
gc_check_after_marks_i(st_data_t k, st_data_t v, st_data_t ptr)
{
VALUE obj = k;
struct reflist *refs = (struct reflist *)v;
rb_objspace_t *objspace = (rb_objspace_t *)ptr;
/* object should be marked or oldgen */
if (!RVALUE_MARKED(objspace, obj)) {
fprintf(stderr, "gc_check_after_marks_i: %s is not marked and not oldgen.\n", rb_obj_info(obj));
fprintf(stderr, "gc_check_after_marks_i: %p is referred from ", (void *)obj);
reflist_dump(refs);
if (reflist_referred_from_machine_context(refs)) {
fprintf(stderr, " (marked from machine stack).\n");
/* marked from machine context can be false positive */
}
else {
objspace->rgengc.error_count++;
fprintf(stderr, "\n");
}
}
return ST_CONTINUE;
}
static void
gc_marks_check(rb_objspace_t *objspace, st_foreach_callback_func *checker_func, const char *checker_name)
{
MALLOC_COUNTERS_LOCK(objspace);
struct gc_malloc_bytes saved_malloc = {
.malloc = gc_counter_load_relaxed(&objspace->malloc_counters.counters.malloc),
.free = gc_counter_load_relaxed(&objspace->malloc_counters.counters.free),
.malloc_at_last_gc = gc_counter_load_relaxed(&objspace->malloc_counters.counters.malloc_at_last_gc),
.free_at_last_gc = gc_counter_load_relaxed(&objspace->malloc_counters.counters.free_at_last_gc),
};
#if RGENGC_ESTIMATE_OLDMALLOC
size_t saved_oldmalloc_increase = gc_oldmalloc_increase_load(objspace);
gc_counter_t saved_freed_in_gc = gc_counter_load_relaxed(&objspace->malloc_counters.freed_in_gc);
#endif
MALLOC_COUNTERS_UNLOCK(objspace);
VALUE already_disabled = rb_objspace_gc_disable(objspace);
objspace->rgengc.allrefs_table = objspace_allrefs(objspace);
if (checker_func) {
st_foreach(objspace->rgengc.allrefs_table, checker_func, (st_data_t)objspace);
}
if (objspace->rgengc.error_count > 0) {
#if RGENGC_CHECK_MODE >= 5
allrefs_dump(objspace);
#endif
if (checker_name) rb_bug("%s: GC has problem.", checker_name);
}
objspace_allrefs_destruct(objspace->rgengc.allrefs_table);
objspace->rgengc.allrefs_table = 0;
if (already_disabled == Qfalse) rb_objspace_gc_enable(objspace);
MALLOC_COUNTERS_LOCK(objspace);
gc_counter_store_release(&objspace->malloc_counters.counters.malloc, saved_malloc.malloc);
gc_counter_store_release(&objspace->malloc_counters.counters.free, saved_malloc.free);
gc_counter_store_release(&objspace->malloc_counters.counters.malloc_at_last_gc, saved_malloc.malloc_at_last_gc);
gc_counter_store_release(&objspace->malloc_counters.counters.free_at_last_gc, saved_malloc.free_at_last_gc);
#if RGENGC_ESTIMATE_OLDMALLOC
RUBY_ATOMIC_SIZE_EXCHANGE(objspace->malloc_counters.oldmalloc_increase, saved_oldmalloc_increase);
gc_counter_store_release(&objspace->malloc_counters.freed_in_gc, saved_freed_in_gc);
#endif
MALLOC_COUNTERS_UNLOCK(objspace);
}
#endif /* RGENGC_CHECK_MODE >= 4 */
struct verify_internal_consistency_struct {
rb_objspace_t *objspace;
/* True only while the world is stopped: a GC.verify holding the VM lock and barrier,
* or a global GC. Cross-objspace checks (walking every objspace's pages) are sound
* only then. */
bool world_stopped;
int err_count;
size_t live_object_count;
size_t zombie_object_count;
VALUE parent;
bool parent_shareable;
size_t old_object_count;
size_t remembered_shady_count;
};
static void
check_generation_i(const VALUE child, void *ptr)
{
struct verify_internal_consistency_struct *data = (struct verify_internal_consistency_struct *)ptr;
const VALUE parent = data->parent;
if (RGENGC_CHECK_MODE) GC_ASSERT(RVALUE_OLD_P(data->objspace, parent));
/* A cross-objspace edge is kept alive by the shareable/shref mechanism and is not
* tracked in this objspace's remembered set. */
if (GET_HEAP_OBJSPACE(child) != data->objspace) return;
/* Once the process goes multi-Ractor, the shareable world is managed by pinning and
* shrefs rather than by the remembered set: the pinned walk at the end of a mark
* re-marks every shareable object (and its shref'd children) each local cycle, and a
* global GC rebuilds the generation state. So the generational old->young invariant
* does not hold when either endpoint is shareable: an old constcache, cc_table or
* interned string pointing at a core class that is young after a global GC is the
* typical false positive. That state outlives the return to a single Ractor until
* the next major (an old shareable singleton class pointing at a young
* attached_object, say), so the test uses rb_gc_ever_multi_ractor_p(), which stays
* true forever once multiple Ractors existed. A program that never goes multi keeps
* the strict check, and ASAN catches what is left. */
if (rb_gc_ever_multi_ractor_p() &&
(MARKED_IN_BITMAP(GET_HEAP_SHAREABLE_BITS(parent), parent) ||
MARKED_IN_BITMAP(GET_HEAP_SHAREABLE_BITS(child), child))) {
return;
}
if (!RVALUE_OLD_P(data->objspace, child)) {
/* A young shareable child is pinned and kept alive by the local GC (only a
* global GC collects it), so it survives even when the old parent does not
* remember it. It is outside the generational remembered set, so exclude it
* from the old->young check. */
if (!RVALUE_REMEMBERED(data->objspace, parent) &&
!RVALUE_REMEMBERED(data->objspace, child) &&
!RVALUE_UNCOLLECTIBLE(data->objspace, child) &&
!RB_FL_TEST_RAW(child, RUBY_FL_SHAREABLE)) {
fprintf(stderr, "verify_internal_consistency_reachable_i: WB miss (O->Y) %s -> %s\n", rb_obj_info(parent), rb_obj_info(child));
data->err_count++;
}
}
}
static void
check_color_i(const VALUE child, void *ptr)
{
struct verify_internal_consistency_struct *data = (struct verify_internal_consistency_struct *)ptr;
const VALUE parent = data->parent;
/* This cycle never marks a foreign child (gc_skip_foreign_object_p) and the write
* barrier is a no-op across objspaces, so its colour says nothing here. */
if (GET_HEAP_OBJSPACE(child) != data->objspace) return;
if (!RVALUE_WB_UNPROTECTED(data->objspace, parent) && RVALUE_WHITE_P(data->objspace, child)) {
fprintf(stderr, "verify_internal_consistency_reachable_i: WB miss (B->W) - %s -> %s\n",
rb_obj_info(parent), rb_obj_info(child));
data->err_count++;
}
}
static void
check_children_i(const VALUE child, void *ptr)
{
struct verify_internal_consistency_struct *data = (struct verify_internal_consistency_struct *)ptr;
/* Fast path: a child in this objspace (99.99% of all edges). */
if (RB_LIKELY(is_pointer_to_heap(data->objspace, (void *)child))) {
if (check_rvalue_consistency_force(data->objspace, child, FALSE) != 0) {
fprintf(stderr, "check_children_i: %s has error (referenced from %s)\n",
rb_obj_info(child), rb_obj_info(data->parent));
data->err_count++;
}
return;
}
/* The remaining cross-objspace check (verify_pointer_in_any_heap_p) walks every
* objspace's pages, sound only with the world stopped: mid-local-GC other Ractors
* change page structures concurrently. The next world-stopped verify re-checks. */
if (!data->world_stopped) return;
/* A non-heap child reaches this callback only when a stale field was followed by a
* plain rb_gc_mark (the dmark of a live but unreachable wrapper, say). Report it and
* keep going rather than aborting. */
if (!verify_pointer_in_any_heap_p((void *)child)) {
/* The graph is in flux mid-merge, so a transient non-heap edge is expected; it
* is re-checked after the merge. */
if (global_objspace->during_absorb) return;
fprintf(stderr, "VERIFY-NOTE: non-heap child %p (from %s)\n",
(void *)child, rb_obj_info(data->parent));
return;
}
if (GET_HEAP_OBJSPACE(child) != data->objspace) {
/* A legal cross-objspace edge either starts at a shareable object or is recorded
* in the child's shref bit (an in-flight send or move payload kept alive across
* its owner's local GC; root_scope_check_i honours the same record). An
* unshareable parent holding an unrecorded foreign unshareable child would be
* invisible to both local GCs. The exception is a box's top_self, which every
* thread's th->top_self points at and which is VM-permanent. Skipped during a
* global GC: it clears every shref bit, so the shref exemption would not fire,
* and its unified exact stop-the-world mark makes the invariant itself moot. */
if (!data->parent_shareable &&
child != rb_gc_vm_top_self() &&
!MARKED_IN_BITMAP(GET_HEAP_SHAREABLE_BITS(child), child) &&
!MARKED_IN_BITMAP(GET_HEAP_SHREF_BITS(child), child) &&
!rb_gc_impl_during_global_gc_p(data->objspace) &&
!global_objspace->during_absorb) {
fprintf(stderr, "check_children_i: containment violation: "
"unshareable %s (objspace %p) -> foreign unshareable %s (objspace %p)\n",
rb_obj_info(data->parent), (void *)data->objspace,
rb_obj_info(child), (void *)GET_HEAP_OBJSPACE(child));
data->err_count++;
}
/* The remaining per-objspace sanity rules belong to the owner. */
return;
}
}
/* Whether a heap slot currently holds a live object. Returns false for empty
* (T_NONE), moved (T_MOVED), and zombie (T_ZOMBIE) slots, and for garbage
* objects about to be swept. */
static bool
gc_slot_live_object_p(rb_objspace_t *objspace, VALUE obj)
{
switch (BUILTIN_TYPE(obj)) {
case T_NONE:
case T_MOVED:
case T_ZOMBIE:
return false;
default:
return !rb_gc_impl_garbage_object_p(objspace, obj);
}
}
/* Verifier only: does ptr point at a live slot in any objspace? The caller holds the VM
* lock and the barrier, so page_index is stable. */
static bool
verify_pointer_in_any_heap_p(const void *ptr)
{
return gc_global_pointer_to_heap_p(ptr);
}
/* An exact root of the calling Ractor may only point at a shareable object, its own
* objspace, or an in-flight payload with a recorded shref. Exempt: the conservative
* machine scan (stale slots) and the VM-global containers that are cross-rooted by
* design (every objspace scans them; the marker skips foreign entries). */
static void
root_scope_check_i(const char *category, VALUE obj, void *ptr)
{
struct verify_internal_consistency_struct *data = ptr;
if (RB_SPECIAL_CONST_P(obj)) return;
/* This check walks every objspace (verify_pointer_in_any_heap_p), so it is sound
* only with the world stopped; a mid-local-GC verify races with other Ractors'
* lock-free allocation. */
if (!data->world_stopped) return;
/* Mid-merge the VM-global root tables still point at the unmerged source (transient
* non-heap or foreign roots); re-checked after the merge. */
if (global_objspace->during_absorb) return;
if (strcmp(category, "machine_context") == 0 ||
strcmp(category, "vm_registered_objects") == 0 ||
strcmp(category, "end_proc") == 0 ||
strcmp(category, "trap_list") == 0) {
return;
}
if (!verify_pointer_in_any_heap_p((void *)obj)) {
fprintf(stderr, "root_scope_check_i: root category \"%s\" names a non-heap pointer %p\n",
category, (void *)obj);
data->err_count++;
return;
}
if (GET_HEAP_OBJSPACE(obj) == data->objspace) return;
if (MARKED_IN_BITMAP(GET_HEAP_SHAREABLE_BITS(obj), obj)) return;
if (MARKED_IN_BITMAP(GET_HEAP_SHREF_BITS(obj), obj)) return;
if (obj == rb_gc_vm_top_self()) return; /* VM-permanent (see check_children_i) */
fprintf(stderr, "root_scope_check_i: root category \"%s\" names a foreign "
"unshareable without a shref record: %s (owner %p, self %p)\n",
category, rb_obj_info(obj),
(void *)GET_HEAP_OBJSPACE(obj), (void *)data->objspace);
data->err_count++;
}
static int
verify_internal_consistency_i(void *page_start, void *page_end, size_t stride,
struct verify_internal_consistency_struct *data)
{
VALUE obj;
rb_objspace_t *objspace = data->objspace;
for (obj = (VALUE)page_start; obj != (VALUE)page_end; obj += stride) {
asan_unpoisoning_object(obj) {
bool sh_bit = MARKED_IN_BITMAP(GET_HEAP_SHAREABLE_BITS(obj), obj) != 0;
bool sr_bit = MARKED_IN_BITMAP(GET_HEAP_SHREF_BITS(obj), obj) != 0;
if (gc_slot_live_object_p(objspace, obj)) {
/* count objects */
data->live_object_count++;
data->parent = obj;
data->parent_shareable = sh_bit;
/* Bitmap invariants: a page's shareable bit matches FL_SHAREABLE
* exactly, and a shref record only ever points at an unshareable
* object. */
if (sh_bit != !!RB_FL_TEST_RAW(obj, RUBY_FL_SHAREABLE)) {
fprintf(stderr, "verify_internal_consistency_i: shareable bit %d "
"disagrees with FL_SHAREABLE on %s\n", (int)sh_bit, rb_obj_info(obj));
data->err_count++;
}
if (sr_bit && sh_bit) {
fprintf(stderr, "verify_internal_consistency_i: shref bit on a shareable: %s\n",
rb_obj_info(obj));
data->err_count++;
}
/* Normally, we don't expect T_MOVED objects to be in the heap.
* But they can stay alive on the stack, */
if (!gc_object_moved_p(objspace, obj)) {
/* moved slots don't have children */
rb_objspace_reachable_objects_from_unlocked(obj, check_children_i, (void *)data);
}
/* check health of children */
if (RVALUE_OLD_P(objspace, obj)) data->old_object_count++;
if (RVALUE_WB_UNPROTECTED(objspace, obj) && RVALUE_UNCOLLECTIBLE(objspace, obj)) data->remembered_shady_count++;
if (!is_marking(objspace) && RVALUE_OLD_P(objspace, obj)) {
/* reachable objects from an oldgen object should be old or (young with remember) */
data->parent = obj;
rb_objspace_reachable_objects_from_unlocked(obj, check_generation_i, (void *)data);
}
if (!is_marking(objspace) && rb_gc_obj_shareable_p(obj)) {
rb_gc_verify_shareable(obj);
}
if (is_incremental_marking(objspace)) {
if (RVALUE_BLACK_P(objspace, obj)) {
/* reachable objects from black objects should be black or grey objects */
data->parent = obj;
rb_objspace_reachable_objects_from_unlocked(obj, check_color_i, (void *)data);
}
}
}
else {
/* A freed slot must not carry its old pin bit into the next object born
* there (a dead object not swept yet legitimately keeps it until the
* sweep arrives). */
if (BUILTIN_TYPE(obj) == T_NONE && (sh_bit || sr_bit)) {
fprintf(stderr, "verify_internal_consistency_i: T_NONE slot carries "
"shareable=%d shref=%d bits\n", (int)sh_bit, (int)sr_bit);
data->err_count++;
}
if (BUILTIN_TYPE(obj) == T_ZOMBIE) {
data->zombie_object_count++;
if ((RBASIC(obj)->flags & ~ZOMBIE_OBJ_KEPT_FLAGS) != T_ZOMBIE) {
fprintf(stderr, "verify_internal_consistency_i: T_ZOMBIE has extra flags set: %s\n",
rb_obj_info(obj));
data->err_count++;
}
if (!!FL_TEST(obj, FL_FINALIZE) != !!st_is_member(finalizer_table, obj)) {
fprintf(stderr, "verify_internal_consistency_i: FL_FINALIZE %s but %s finalizer_table: %s\n",
FL_TEST(obj, FL_FINALIZE) ? "set" : "not set", st_is_member(finalizer_table, obj) ? "in" : "not in",
rb_obj_info(obj));
data->err_count++;
}
}
}
}
}
return 0;
}
static int
gc_verify_heap_page(rb_objspace_t *objspace, struct heap_page *page, VALUE obj)
{
unsigned int has_remembered_shady = FALSE;
unsigned int has_remembered_old = FALSE;
int remembered_old_objects = 0;
int free_objects = 0;
int zombie_objects = 0;
short slot_size = page->slot_size;
uintptr_t start = (uintptr_t)page->start;
uintptr_t end = start + page->total_slots * slot_size;
for (uintptr_t ptr = start; ptr < end; ptr += slot_size) {
VALUE val = (VALUE)ptr;
asan_unpoisoning_object(val) {
enum ruby_value_type type = BUILTIN_TYPE(val);
if (type == T_NONE) free_objects++;
if (type == T_ZOMBIE) zombie_objects++;
if (RVALUE_PAGE_UNCOLLECTIBLE(page, val) && RVALUE_PAGE_WB_UNPROTECTED(page, val)) {
has_remembered_shady = TRUE;
}
if (RVALUE_PAGE_MARKING(page, val)) {
has_remembered_old = TRUE;
remembered_old_objects++;
}
}
}
if (!is_incremental_marking(objspace) &&
page->flags.has_remembered_objects == FALSE && has_remembered_old == TRUE) {
for (uintptr_t ptr = start; ptr < end; ptr += slot_size) {
VALUE val = (VALUE)ptr;
if (RVALUE_PAGE_MARKING(page, val)) {
fprintf(stderr, "marking -> %s\n", rb_obj_info(val));
}
}
rb_bug("page %p's has_remembered_objects should be false, but there are remembered old objects (%d). %s",
(void *)page, remembered_old_objects, obj ? rb_obj_info(obj) : "");
}
if (page->flags.has_uncollectible_wb_unprotected_objects == FALSE && has_remembered_shady == TRUE) {
rb_bug("page %p's has_remembered_shady should be false, but there are remembered shady objects. %s",
(void *)page, obj ? rb_obj_info(obj) : "");
}
if (0) {
/* free_slots may not equal to free_objects */
if (page->free_slots != free_objects) {
rb_bug("page %p's free_slots should be %d, but %d", (void *)page, page->free_slots, free_objects);
}
}
if (page->final_slots != zombie_objects) {
rb_bug("page %p's final_slots should be %d, but %d", (void *)page, page->final_slots, zombie_objects);
}
return remembered_old_objects;
}
static int
gc_verify_heap_pages_(rb_objspace_t *objspace, struct ccan_list_head *head)
{
int remembered_old_objects = 0;
struct heap_page *page = 0;
ccan_list_for_each(head, page, page_node) {
asan_unlock_freelist(page);
struct free_region *region = page->free_region;
while (region) {
VALUE vp = (VALUE)region;
rb_asan_unpoison_object(vp, false);
if (BUILTIN_TYPE(vp) != T_NONE) {
fprintf(stderr, "free region head expected to be T_NONE but was: %s\n", rb_obj_info(vp));
}
struct free_region *next = region->next;
rb_asan_poison_object(vp);
region = next;
}
asan_lock_freelist(page);
if (page->flags.has_remembered_objects == FALSE) {
remembered_old_objects += gc_verify_heap_page(objspace, page, Qfalse);
}
}
return remembered_old_objects;
}
static int
gc_verify_heap_pages(rb_objspace_t *objspace)
{
int remembered_old_objects = 0;
for (int i = 0; i < HEAP_COUNT; i++) {
remembered_old_objects += gc_verify_heap_pages_(objspace, &((&heaps[i])->pages));
}
return remembered_old_objects;
}
static void
verify_registered_addr(VALUE *slot, VALUE initial_value, void *owner_objspace, void *d)
{
struct verify_internal_consistency_struct *data = d;
VALUE v = *slot;
/* Conservative registration permits uninitialized data and pre-registration
* values; only a store made after registration is a violation. */
if (v == initial_value) return;
if (SPECIAL_CONST_P(v)) return;
if (!verify_pointer_in_any_heap_p((void *)v)) return;
bool live = false;
asan_unpoisoning_object(v) {
live = BUILTIN_TYPE(v) != T_NONE && BUILTIN_TYPE(v) != T_ZOMBIE;
}
if (!live) return;
rb_objspace_t *value_objspace = GET_HEAP_OBJSPACE(v);
if (value_objspace == (rb_objspace_t *)owner_objspace) return;
/* Join and orphan handling move a registration to the inheritor before the
* source objspace merge; a global GC scans every registry while the zombie
* exists, so this is a safe transient exemption. */
if (value_objspace->zombie_pages != 0) return;
if (value_objspace->flags.during_postmortem) return;
if (MARKED_IN_BITMAP(GET_HEAP_SHAREABLE_BITS(v), v)) return;
if (MARKED_IN_BITMAP(GET_HEAP_SHREF_BITS(v), v)) return;
/* When multiple Ractors register one address, ownership by any registrant is
* enough to root the value. */
if (rb_gc_registered_addr_owned_by_registrant_p(slot, value_objspace)) return;
fprintf(stderr, "registered address %p changed since registration to an unshareable object owned by another Ractor: %s\n",
(void *)slot, rb_obj_info(v));
data->err_count++;
}
static void
gc_verify_internal_consistency_(rb_objspace_t *objspace, bool world_stopped)
{
struct verify_internal_consistency_struct data = {0};
data.objspace = objspace;
data.world_stopped = world_stopped;
gc_report(5, objspace, "gc_verify_internal_consistency: start\n");
/* check relations */
for (size_t i = 0; i < rb_darray_size(objspace->heap_pages.sorted); i++) {
struct heap_page *page = rb_darray_get(objspace->heap_pages.sorted, i);
short slot_size = page->slot_size;
uintptr_t start = (uintptr_t)page->start;
uintptr_t end = start + page->total_slots * slot_size;
verify_internal_consistency_i((void *)start, (void *)end, slot_size, &data);
}
/* Check the calling Ractor's root scoping (only when verifying the current
* objspace). Skipped during a global GC, which deliberately spans every Ractor's
* roots and legitimately reaches foreign objects: containment does not apply. */
if (!rb_gc_single_objspace_p() && objspace == rb_gc_get_objspace() &&
!rb_gc_impl_during_global_gc_p(objspace)) {
rb_objspace_reachable_objects_from_root(root_scope_check_i, &data);
}
if (data.world_stopped && !global_objspace->during_absorb) {
rb_gc_each_registered_addr(verify_registered_addr, &data);
}
if (data.err_count != 0) {
#if RGENGC_CHECK_MODE >= 5
objspace->rgengc.error_count = data.err_count;
gc_marks_check(objspace, NULL, NULL);
allrefs_dump(objspace);
#endif
rb_bug("gc_verify_internal_consistency: found internal inconsistency.");
}
/* check heap_page status */
gc_verify_heap_pages(objspace);
/* check counters */
if (!is_lazy_sweeping(objspace) &&
!finalizing &&
!rb_gc_multi_ractor_p()) {
if (objspace_live_slots(objspace) != data.live_object_count) {
fprintf(stderr, "heap_pages_final_slots: %"PRIdSIZE", total_freed_objects: %"PRIdSIZE"\n",
total_final_slots_count(objspace), total_freed_objects(objspace));
rb_bug("inconsistent live slot number: expect %"PRIuSIZE", but %"PRIuSIZE".",
objspace_live_slots(objspace), data.live_object_count);
}
}
if (!is_marking(objspace)) {
if (objspace->rgengc.old_objects != data.old_object_count) {
rb_bug("inconsistent old slot number: expect %"PRIuSIZE", but %"PRIuSIZE".",
objspace->rgengc.old_objects, data.old_object_count);
}
if (objspace->rgengc.uncollectible_wb_unprotected_objects != data.remembered_shady_count) {
rb_bug("inconsistent number of wb unprotected objects: expect %"PRIuSIZE", but %"PRIuSIZE".",
objspace->rgengc.uncollectible_wb_unprotected_objects, data.remembered_shady_count);
}
}
if (!finalizing) {
size_t list_count = 0;
{
VALUE z = heap_pages_deferred_final;
while (z) {
list_count++;
z = RZOMBIE(z)->next;
}
}
if (total_final_slots_count(objspace) != data.zombie_object_count ||
total_final_slots_count(objspace) != list_count) {
rb_bug("inconsistent finalizing object count:\n"
" expect %"PRIuSIZE"\n"
" but %"PRIuSIZE" zombies\n"
" heap_pages_deferred_final list has %"PRIuSIZE" items.",
total_final_slots_count(objspace),
data.zombie_object_count,
list_count);
}
}
gc_report(5, objspace, "gc_verify_internal_consistency: OK\n");
}
/* The `during_gc` macro expands a bare identifier to `objspace->flags.during_gc`, so a
* foreign objspace's flag cannot be written directly; these helpers reach it through the
* `objspace` argument. */
static inline unsigned int
gc_during_gc_get(const rb_objspace_t *objspace)
{
return during_gc;
}
static inline void
gc_during_gc_set(rb_objspace_t *objspace, unsigned int v)
{
during_gc = v;
}
/* Run the check with during_gc cleared in both the verified objspace and the current
* Ractor's: rb_objspace_reachable_objects_from() decides on rb_gc_get_objspace(), and
* under a global GC the driver verifies foreign objspaces, so the driver's during_gc
* needs clearing too (a no-op when cur == objspace). */
static void
gc_verify_internal_consistency_body(rb_objspace_t *objspace, bool world_stopped)
{
const unsigned int prev_during_gc = during_gc;
during_gc = FALSE; // stop gc here
rb_objspace_t *const cur = rb_gc_get_objspace();
const unsigned int prev_cur_during_gc = (cur != objspace) ? gc_during_gc_get(cur) : 0;
if (cur != objspace) gc_during_gc_set(cur, FALSE);
{
gc_verify_internal_consistency_(objspace, world_stopped);
}
if (cur != objspace) gc_during_gc_set(cur, prev_cur_during_gc);
during_gc = prev_during_gc;
}
static void
gc_verify_internal_consistency(void *objspace_ptr)
{
rb_objspace_t *objspace = objspace_ptr;
/* Called mid-GC, take neither the VM lock nor the barrier: waiting would join a
* pending global barrier mid-collection (a GC must never take the VM lock) and let
* the global GC sweep the heap this mark is walking. The barrier is unnecessary
* anyway; the objspace is single-writer, this verify runs on its owner thread, and
* the global driver that sets during_gc everywhere already holds both. */
if (during_gc) {
/* The world is stopped only when the global GC's driver runs this while holding
* the barrier; a non-main Ractor's local GC does not stop other Ractors. */
gc_verify_internal_consistency_body(objspace, rb_gc_impl_during_global_gc_p(objspace));
return;
}
unsigned int lev = RB_GC_VM_LOCK();
{
rb_gc_vm_barrier(); // stop other ractors
gc_verify_internal_consistency_body(objspace, true); // holding the barrier, so walking every objspace is sound
}
RB_GC_VM_UNLOCK(lev);
}
static void
heap_move_pooled_pages_to_free_pages(rb_heap_t *heap)
{
if (heap->pooled_pages) {
if (heap->free_pages) {
struct heap_page *free_pages_tail = heap->free_pages;
while (free_pages_tail->free_next) {
free_pages_tail = free_pages_tail->free_next;
}
free_pages_tail->free_next = heap->pooled_pages;
}
else {
heap->free_pages = heap->pooled_pages;
}
heap->pooled_pages = NULL;
}
}
static int
gc_remember_unprotected(rb_objspace_t *objspace, VALUE obj)
{
struct heap_page *page = GET_HEAP_PAGE(obj);
bits_t *uncollectible_bits = &page->uncollectible_bits[0];
if (!MARKED_IN_BITMAP(uncollectible_bits, obj)) {
page->flags.has_uncollectible_wb_unprotected_objects = TRUE;
MARK_IN_BITMAP(uncollectible_bits, obj);
/* Like RVALUE_PAGE_OLD_UNCOLLECTIBLE_SET, count it in the object's own objspace. */
page->objspace->rgengc.uncollectible_wb_unprotected_objects++;
#if RGENGC_PROFILE > 0
objspace->profile.total_remembered_shady_object_count++;
#if RGENGC_PROFILE >= 2
objspace->profile.remembered_shady_object_count_types[BUILTIN_TYPE(obj)]++;
#endif
#endif
return TRUE;
}
else {
return FALSE;
}
}
static inline void
gc_marks_wb_unprotected_objects_plane(rb_objspace_t *objspace, uintptr_t p, bits_t bits, short slot_size)
{
if (bits) {
do {
if (bits & 1) {
gc_report(2, objspace, "gc_marks_wb_unprotected_objects: marked shady: %s\n", rb_obj_info((VALUE)p));
GC_ASSERT(RVALUE_WB_UNPROTECTED(objspace, (VALUE)p));
GC_ASSERT(RVALUE_MARKED(objspace, (VALUE)p));
gc_mark_children(objspace, (VALUE)p);
}
p += slot_size;
bits >>= 1;
} while (bits);
}
}
static void
gc_marks_wb_unprotected_objects(rb_objspace_t *objspace, rb_heap_t *heap)
{
struct heap_page *page = 0;
ccan_list_for_each(&heap->pages, page, page_node) {
bits_t *mark_bits = page->mark_bits;
bits_t *wbun_bits = page->wb_unprotected_bits;
uintptr_t p = page->start;
short slot_size = page->slot_size;
int total_slots = page->total_slots;
int bitmap_plane_count = CEILDIV(total_slots, BITS_BITLENGTH);
size_t j;
for (j=0; j<(size_t)bitmap_plane_count; j++) {
bits_t bits = mark_bits[j] & wbun_bits[j];
gc_marks_wb_unprotected_objects_plane(objspace, p, bits, slot_size);
p += BITS_BITLENGTH * slot_size;
}
}
gc_mark_stacked_objects_all(objspace);
}
void
rb_gc_impl_declare_weak_references(void *objspace_ptr, VALUE obj)
{
FL_SET_RAW(obj, RUBY_FL_WEAK_REFERENCE);
}
bool
rb_gc_impl_handle_weak_references_alive_p(void *objspace_ptr, VALUE obj)
{
rb_objspace_t *objspace = objspace_ptr;
/* A local GC cannot decide a foreign object's liveness, so treat it as live; its
* owner or the global GC decides (a global GC's unified mark is exact). */
if (gc_skip_foreign_object_p(objspace, obj)) return true;
bool marked = RVALUE_MARKED(objspace, obj);
if (marked) {
rgengc_check_relation(objspace, obj);
}
return marked;
}
static void
gc_update_weak_references(rb_objspace_t *objspace)
{
VALUE *obj_ptr;
rb_darray_foreach(objspace->weak_references, i, obj_ptr) {
gc_mark_set_parent(objspace, *obj_ptr);
rb_gc_handle_weak_references(*obj_ptr);
gc_mark_set_parent_invalid(objspace);
}
size_t capa = rb_darray_capa(objspace->weak_references);
size_t size = rb_darray_size(objspace->weak_references);
objspace->profile.weak_references_count = size;
rb_darray_clear(objspace->weak_references);
/* If the darray has capacity for more than four times the amount used, we
* shrink it down to half of that capacity. */
if (capa > size * 4) {
rb_darray_resize_capa_without_gc(&objspace->weak_references, size * 2);
}
}
static void
gc_marks_finish(rb_objspace_t *objspace)
{
/* finish incremental GC */
if (is_incremental_marking(objspace)) {
if (RGENGC_CHECK_MODE && is_mark_stack_empty(&objspace->mark_stack) == 0) {
rb_bug("gc_marks_finish: mark stack is not empty (%"PRIdSIZE").",
mark_stack_size(&objspace->mark_stack));
}
mark_roots(objspace, NULL);
while (gc_mark_stacked_objects_incremental(objspace, INT_MAX) == false);
#if RGENGC_CHECK_MODE >= 2
if (gc_verify_heap_pages(objspace) != 0) {
rb_bug("gc_marks_finish (incremental): there are remembered old objects.");
}
#endif
gc_set_incremental_marking(objspace, FALSE);
/* check children of all marked wb-unprotected objects */
for (int i = 0; i < HEAP_COUNT; i++) {
gc_marks_wb_unprotected_objects(objspace, &heaps[i]);
}
}
/* Pin the shareable objects and shrefs ordinary marking missed: a local GC must free
* neither (another objspace may hold them). Running after the full walk makes the
* pin count a retention metric: an upper bound on the garbage only a global GC can
* reclaim. A global GC's exact mark does not pin. (The allrefs comparison of
* RGENGC_CHECK_MODE >= 4 does not model these pins; it reports false positives.)
*
* Running it here rather than with the other roots is also what lets an incremental
* mark run while other objspaces exist. The write barrier bails out on a
* cross-objspace edge, so a store made between two mark steps leaves nothing behind
* but a shareable or shref bit; scanning those bitmaps after the last step picks up
* every bit set during the cycle, which a scan at gc_marks_start would miss. */
objspace->last_cycle_pinned = 0;
if (!rb_gc_single_objspace_p() && !objspace->flags.during_global_gc) {
objspace->last_cycle_pinned = 1;
gc_mark_set_parent_raw(objspace, Qundef, false);
for (int i = 0; i < HEAP_COUNT; i++) {
pinned_roots_mark(objspace, &heaps[i]);
}
/* And everything they keep alive. */
gc_mark_stacked_objects_all(objspace);
}
gc_update_weak_references(objspace);
#if RGENGC_CHECK_MODE >= 4
during_gc = FALSE;
gc_marks_check(objspace, gc_check_after_marks_i, "after_marks");
during_gc = TRUE;
#endif
{
/* Only this objspace's own Ractor allocates from it. The main objspace
* keeps the VM-wide count it has used since before per-Ractor GC. */
const unsigned long ractor_cnt = objspace == global_objspace->main_objspace
? rb_gc_vm_ractor_count() : 1;
const unsigned long r_mul = ractor_cnt > 8 ? 8 : ractor_cnt; // upto 8
size_t total_slots = objspace_available_slots(objspace);
size_t sweep_slots = total_slots - objspace->marked_slots; /* will be swept slots */
size_t max_free_slots = (size_t)(total_slots * gc_params.heap_free_slots_max_ratio);
size_t min_free_slots = (size_t)(total_slots * gc_params.heap_free_slots_min_ratio);
if (min_free_slots < gc_params.heap_free_slots * r_mul) {
min_free_slots = gc_params.heap_free_slots * r_mul;
}
int full_marking = is_full_marking(objspace);
GC_ASSERT(objspace_available_slots(objspace) >= objspace->marked_slots);
/* Setup freeable slots. */
size_t total_init_slots = 0;
for (int i = 0; i < HEAP_COUNT; i++) {
total_init_slots += (objspace_heap_init_bytes(objspace) / heaps[i].slot_size) * r_mul;
}
if (max_free_slots < total_init_slots) {
max_free_slots = total_init_slots;
}
/* Approximate freeable pages using the average slots-per-pages across all heaps */
if (sweep_slots > max_free_slots) {
size_t excess_slots = sweep_slots - max_free_slots;
size_t total_heap_pages = heap_eden_total_pages(objspace);
heap_pages_freeable_pages = total_heap_pages > 0
? excess_slots * total_heap_pages / total_slots
: 0;
}
else {
heap_pages_freeable_pages = 0;
}
if (objspace->heap_pages.allocatable_bytes == 0 && sweep_slots < min_free_slots) {
if (!full_marking && sweep_slots < min_free_slots * 7 / 8) {
if (objspace->profile.count - objspace->rgengc.last_major_gc < RVALUE_OLD_AGE) {
full_marking = TRUE;
}
else {
gc_report(1, objspace, "gc_marks_finish: next is full GC!!)\n");
gc_needs_major_flags |= GPR_FLAG_MAJOR_BY_NOFREE;
}
}
if (full_marking) {
heap_allocatable_bytes_expand(objspace, NULL, sweep_slots, total_slots, heaps[0].slot_size);
}
}
if (full_marking) {
/* See the comment about RUBY_GC_HEAP_OLDOBJECT_LIMIT_FACTOR */
const double r = gc_params.oldobject_limit_factor;
objspace->rgengc.uncollectible_wb_unprotected_objects_limit = MAX(
(size_t)(objspace->rgengc.uncollectible_wb_unprotected_objects * r),
(size_t)(objspace->rgengc.old_objects * gc_params.uncollectible_wb_unprotected_objects_limit_ratio)
);
objspace->rgengc.old_objects_limit = (size_t)(objspace->rgengc.old_objects * r);
}
if (objspace->rgengc.uncollectible_wb_unprotected_objects > objspace->rgengc.uncollectible_wb_unprotected_objects_limit) {
gc_needs_major_flags |= GPR_FLAG_MAJOR_BY_SHADY;
}
if (objspace->rgengc.old_objects > objspace->rgengc.old_objects_limit) {
gc_needs_major_flags |= GPR_FLAG_MAJOR_BY_OLDGEN;
}
gc_report(1, objspace, "gc_marks_finish (marks %"PRIdSIZE" objects, "
"old %"PRIdSIZE" objects, total %"PRIdSIZE" slots, "
"sweep %"PRIdSIZE" slots, allocatable %"PRIdSIZE" bytes, next GC: %s)\n",
objspace->marked_slots, objspace->rgengc.old_objects, objspace_available_slots(objspace), sweep_slots, objspace->heap_pages.allocatable_bytes,
gc_needs_major_flags ? "major" : "minor");
}
// TODO: refactor so we don't need to call this
rb_ractor_finish_marking(is_full_marking(objspace));
gc_event_hook(objspace, RUBY_INTERNAL_EVENT_GC_END_MARK);
}
static bool
gc_compact_heap_cursors_met_p(rb_heap_t *heap)
{
return heap->sweeping_page == heap->compact_cursor;
}
static rb_heap_t *
gc_compact_destination_pool(rb_objspace_t *objspace, rb_heap_t *src_pool, VALUE obj)
{
size_t obj_size = rb_gc_obj_optimal_size(obj);
if (obj_size == 0) {
return src_pool;
}
GC_ASSERT(rb_gc_impl_size_allocatable_p(obj_size));
size_t idx = heap_idx_for_size(obj_size);
return &heaps[idx];
}
static bool
gc_compact_move(rb_objspace_t *objspace, rb_heap_t *heap, VALUE src)
{
GC_ASSERT(BUILTIN_TYPE(src) != T_MOVED);
GC_ASSERT(gc_is_moveable_obj(objspace, src));
rb_heap_t *dest_pool = gc_compact_destination_pool(objspace, heap, src);
if (gc_compact_heap_cursors_met_p(dest_pool)) {
return dest_pool != heap;
}
while (!try_move(objspace, dest_pool, dest_pool->free_pages, src)) {
struct gc_sweep_context ctx = {
.page = dest_pool->sweeping_page,
.final_slots = 0,
.freed_slots = 0,
.empty_slots = 0,
};
/* The page of src could be partially compacted, so it may contain
* T_MOVED. Sweeping a page may read objects on this page, so we
* need to lock the page. */
lock_page_body(objspace, GET_PAGE_BODY(src));
gc_sweep_page(objspace, dest_pool, &ctx);
unlock_page_body(objspace, GET_PAGE_BODY(src));
if (dest_pool->sweeping_page->free_slots > 0) {
heap_add_freepage(dest_pool, dest_pool->sweeping_page);
}
dest_pool->sweeping_page = ccan_list_next(&dest_pool->pages, dest_pool->sweeping_page, page_node);
if (gc_compact_heap_cursors_met_p(dest_pool)) {
return dest_pool != heap;
}
}
return true;
}
static bool
gc_compact_plane(rb_objspace_t *objspace, rb_heap_t *heap, uintptr_t p, bits_t bitset, struct heap_page *page)
{
short slot_size = page->slot_size;
do {
VALUE vp = (VALUE)p;
GC_ASSERT(vp % sizeof(VALUE) == 0);
if (bitset & 1) {
objspace->rcompactor.considered_count_table[BUILTIN_TYPE(vp)]++;
if (gc_is_moveable_obj(objspace, vp)) {
if (!gc_compact_move(objspace, heap, vp)) {
//the cursors met. bubble up
return false;
}
}
}
p += slot_size;
bitset >>= 1;
} while (bitset);
return true;
}
// Iterate up all the objects in page, moving them to where they want to go
static bool
gc_compact_page(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *page)
{
GC_ASSERT(page == heap->compact_cursor);
bits_t *mark_bits, *pin_bits;
bits_t bitset;
uintptr_t p = page->start;
short slot_size = page->slot_size;
int total_slots = page->total_slots;
int bitmap_plane_count = CEILDIV(total_slots, BITS_BITLENGTH);
mark_bits = page->mark_bits;
pin_bits = page->pinned_bits;
for (int j = 0; j < bitmap_plane_count; j++) {
// objects that can be moved are marked and not pinned
bitset = (mark_bits[j] & ~pin_bits[j]);
if (bitset) {
if (!gc_compact_plane(objspace, heap, (uintptr_t)p, bitset, page))
return false;
}
p += BITS_BITLENGTH * slot_size;
}
return true;
}
static bool
gc_compact_all_compacted_p(rb_objspace_t *objspace)
{
for (int i = 0; i < HEAP_COUNT; i++) {
rb_heap_t *heap = &heaps[i];
if (heap->total_pages > 0 &&
!gc_compact_heap_cursors_met_p(heap)) {
return false;
}
}
return true;
}
/* Compaction's move phase: relocate this objspace's movable objects and leave T_MOVED
* forwarding behind without updating references yet. A global GC calls this for every
* objspace before updating any of them (two phases), so a cross-objspace reference to a
* moved object is rewritten exactly once, after all forwarding exists. */
static void
gc_compact_relocate(rb_objspace_t *objspace)
{
gc_compact_start(objspace);
while (!gc_compact_all_compacted_p(objspace)) {
for (int i = 0; i < HEAP_COUNT; i++) {
rb_heap_t *heap = &heaps[i];
if (gc_compact_heap_cursors_met_p(heap)) {
continue;
}
struct heap_page *start_page = heap->compact_cursor;
if (!gc_compact_page(objspace, heap, start_page)) {
lock_page_body(objspace, start_page->body);
continue;
}
// If we get here, we've finished moving all objects on the compact_cursor page
// So we can lock it and move the cursor on to the next one.
lock_page_body(objspace, start_page->body);
heap->compact_cursor = ccan_list_prev(&heap->pages, heap->compact_cursor, page_node);
}
}
}
static void
gc_sweep_compact(rb_objspace_t *objspace)
{
gc_compact_relocate(objspace);
/* A compacting global GC defers the finish (reference update) to the second phase,
* after every objspace has been relocated. */
if (!global_objspace->global_gc.compacting) {
gc_compact_finish(objspace);
}
}
static void
gc_marks_rest(rb_objspace_t *objspace)
{
gc_report(1, objspace, "gc_marks_rest\n");
for (int i = 0; i < HEAP_COUNT; i++) {
(&heaps[i])->pooled_pages = NULL;
}
if (is_incremental_marking(objspace)) {
while (gc_mark_stacked_objects_incremental(objspace, INT_MAX) == FALSE);
}
else {
gc_mark_stacked_objects_all(objspace);
}
gc_marks_finish(objspace);
}
static bool
gc_marks_step(rb_objspace_t *objspace, size_t slots)
{
bool marking_finished = false;
GC_ASSERT(is_marking(objspace));
if (gc_mark_stacked_objects_incremental(objspace, slots)) {
gc_marks_finish(objspace);
marking_finished = true;
}
return marking_finished;
}
static bool
gc_marks_continue(rb_objspace_t *objspace, rb_heap_t *heap)
{
GC_ASSERT(dont_gc_val() == FALSE || objspace->profile.latest_gc_info & GPR_FLAG_METHOD);
bool marking_finished = true;
gc_marking_enter(objspace);
if (heap->free_pages) {
gc_report(2, objspace, "gc_marks_continue: has pooled pages");
marking_finished = gc_marks_step(objspace, objspace->rincgc.step_slots);
}
else {
gc_report(2, objspace, "gc_marks_continue: no more pooled pages (stack depth: %"PRIdSIZE").\n",
mark_stack_size(&objspace->mark_stack));
heap->force_incremental_marking_finish_count++;
gc_marks_rest(objspace);
}
gc_marking_exit(objspace);
return marking_finished;
}
/* Mark the following as roots of this objspace.
* - Every shareable object: another objspace may hold the only reference, invisible to a
* local GC. Marking them rather than skipping them in the sweep preserves the
* generational invariants (a pinned object ages and gets promoted like any live one).
* Only a global GC decides that a shareable object is dead.
* - Every shref (an unshareable object referenced from a shareable one): the referring
* shareable object can live in another objspace or in an in-flight message queue. The
* write barrier maintains them.
* Skipped while the VM has a single Ractor: a local GC is then a whole-world GC and
* shareable objects may die normally. */
static void
pinned_roots_mark(rb_objspace_t *objspace, rb_heap_t *heap)
{
struct heap_page *page = NULL;
/* Runs before mark_roots, so rgengc_check_relation sees a valid (absent) parent rather
* than the poison left by the previous GC. */
gc_mark_set_parent_raw(objspace, Qundef, false);
/* A local GC never frees or traverses a shareable object, and keeps its unshareable
* children alive through their shref bits, so:
* - a shareable object only gets its mark bit set (like an old object), which keeps
* the sweep off it, and is not traversed;
* - a shref is marked and traversed, like a remembered old->young target: without
* that, the referring shareable object is never walked and it would look
* unreachable.
* Objects can become shareable between GCs, so this pass scans the bitmaps in every
* mark (gc_marks_finish) instead of maintaining a pin set across the sweep. */
ccan_list_for_each(&heap->pages, page, page_node) {
if (!(page->flags.has_shareable_objects | page->flags.has_shref_objects)) continue;
uintptr_t p = page->start;
short slot_size = page->slot_size;
int total_slots = page->total_slots;
int bitmap_plane_count = CEILDIV(total_slots, BITS_BITLENGTH);
for (int j = 0; j < bitmap_plane_count; j++) {
bits_t sr_bits = page->shref_bits[j];
/* Only the pins ordinary marking left unmarked need work here: an already
* marked object (reached by traversal, or pre-marked because it is old) is a
* no-op in gc_mark_set, so skip visiting it. */
bits_t bitset = (page->shareable_bits[j] | sr_bits) & ~page->mark_bits[j];
uintptr_t pp = p;
while (bitset) {
if (bitset & 1) {
VALUE obj = (VALUE)pp;
asan_unpoisoning_object(obj) {
switch (BUILTIN_TYPE(obj)) {
case T_NONE:
case T_ZOMBIE:
case T_MOVED:
/* A dead slot (a zombie awaiting its finalizer) is not a root. */
break;
default:
gc_report(2, objspace, "pinned_roots_mark: mark %s\n", rb_obj_info(obj));
if (sr_bits & 1) {
gc_mark(objspace, obj); /* shref: root + traverse */
}
else if (gc_mark_set(objspace, obj)) {
gc_aging(objspace, obj); /* shareable: mark, no traverse */
/* Pin as well when compaction runs alongside: if a shareable
* object moved, the C-struct slots of other Ractors (a
* port in sync, say) are not updated and go stale. */
gc_pin(objspace, obj);
}
break;
}
}
}
pp += slot_size;
bitset >>= 1;
sr_bits >>= 1;
}
p += BITS_BITLENGTH * slot_size;
}
}
}
static void
gc_marks_start(rb_objspace_t *objspace, int full_mark)
{
/* start marking */
gc_report(1, objspace, "gc_marks_start: (%s)\n", full_mark ? "full" : "minor");
gc_mode_transition(objspace, gc_mode_marking);
if (full_mark) {
size_t incremental_marking_steps = (objspace->rincgc.pooled_slots / INCREMENTAL_MARK_STEP_ALLOCATIONS) + 1;
objspace->rincgc.step_slots = (objspace->marked_slots * 2) / incremental_marking_steps;
if (0) fprintf(stderr, "objspace->marked_slots: %"PRIdSIZE", "
"objspace->rincgc.pooled_page_num: %"PRIdSIZE", "
"objspace->rincgc.step_slots: %"PRIdSIZE", \n",
objspace->marked_slots, objspace->rincgc.pooled_slots, objspace->rincgc.step_slots);
objspace->flags.during_minor_gc = FALSE;
if (ruby_enable_autocompact && rb_gc_single_objspace_p()) {
objspace->flags.during_compacting |= TRUE;
}
objspace->profile.major_gc_count++;
objspace->rgengc.uncollectible_wb_unprotected_objects = 0;
objspace->rgengc.old_objects = 0;
objspace->rgengc.last_major_gc = objspace->profile.count;
objspace->marked_slots = 0;
for (int i = 0; i < HEAP_COUNT; i++) {
rb_heap_t *heap = &heaps[i];
gc_bitmaps_clear(objspace, heap, false);
heap_move_pooled_pages_to_free_pages(heap);
if (objspace->flags.during_compacting) {
struct heap_page *page = NULL;
ccan_list_for_each(&heap->pages, page, page_node) {
page->pinned_slots = 0;
}
}
}
}
else {
objspace->flags.during_minor_gc = TRUE;
objspace->marked_slots =
objspace->rgengc.old_objects + objspace->rgengc.uncollectible_wb_unprotected_objects; /* uncollectible objects are marked already */
objspace->profile.minor_gc_count++;
for (int i = 0; i < HEAP_COUNT; i++) {
rgengc_rememberset_mark(objspace, &heaps[i]);
}
}
mark_roots(objspace, NULL);
gc_report(1, objspace, "gc_marks_start: (%s) end, stack in %"PRIdSIZE"\n",
full_mark ? "full" : "minor", mark_stack_size(&objspace->mark_stack));
}
static bool
gc_marks(rb_objspace_t *objspace, int full_mark)
{
gc_marking_enter(objspace);
bool marking_finished = false;
/* setup marking */
gc_marks_start(objspace, full_mark);
if (!is_incremental_marking(objspace)) {
gc_marks_rest(objspace);
marking_finished = true;
}
#if RGENGC_PROFILE > 0
if (gc_prof_record(objspace)) {
gc_profile_record *record = gc_prof_record(objspace);
record->old_objects = objspace->rgengc.old_objects;
}
#endif
gc_marking_exit(objspace);
return marking_finished;
}
/* RGENGC */
static void
gc_report_body(int level, rb_objspace_t *objspace, const char *fmt, ...)
{
if (level <= RGENGC_DEBUG) {
char buf[1024];
FILE *out = stderr;
va_list args;
const char *status = " ";
if (during_gc) {
status = is_full_marking(objspace) ? "+" : "-";
}
else {
if (is_lazy_sweeping(objspace)) {
status = "S";
}
if (is_incremental_marking(objspace)) {
status = "M";
}
}
va_start(args, fmt);
vsnprintf(buf, 1024, fmt, args);
va_end(args);
fprintf(out, "%s|", status);
fputs(buf, out);
}
}
/* bit operations */
static void
rgengc_remembersetbits_set(rb_objspace_t *objspace, VALUE obj)
{
struct heap_page *page = GET_HEAP_PAGE(obj);
bits_t *bits = &page->remembered_bits[0];
/* remembered_bits writers are always serialized: the write barrier only remembers a
* local a (under its Ractor's GVL) and a global GC writes from the driver alone.
* Set the bit before the page flag so a page pending re-scan stays in
* rememberset_mark. */
_MARK_IN_BITMAP(bits, page, obj);
page->flags.has_remembered_objects = TRUE;
}
/* wb, etc */
/* return FALSE if already remembered */
static void
rgengc_remember(rb_objspace_t *objspace, VALUE obj)
{
gc_report(6, objspace, "rgengc_remember: %s %s\n", rb_obj_info(obj),
RVALUE_REMEMBERED(objspace, obj) ? "was already remembered" : "is remembered now");
check_rvalue_consistency(objspace, obj);
if (RGENGC_CHECK_MODE) {
if (RVALUE_WB_UNPROTECTED(objspace, obj)) rb_bug("rgengc_remember: %s is not wb protected.", rb_obj_info(obj));
}
#if RGENGC_PROFILE > 0
if (!RVALUE_REMEMBERED(objspace, obj)) {
if (RVALUE_WB_UNPROTECTED(objspace, obj) == 0) {
objspace->profile.total_remembered_normal_object_count++;
#if RGENGC_PROFILE >= 2
objspace->profile.remembered_normal_object_count_types[BUILTIN_TYPE(obj)]++;
#endif
}
}
#endif /* RGENGC_PROFILE > 0 */
rgengc_remembersetbits_set(objspace, obj);
}
#ifndef PROFILE_REMEMBERSET_MARK
#define PROFILE_REMEMBERSET_MARK 0
#endif
static inline void
rgengc_rememberset_mark_plane(rb_objspace_t *objspace, uintptr_t p, bits_t bitset, short slot_size)
{
if (bitset) {
do {
if (bitset & 1) {
VALUE obj = (VALUE)p;
gc_report(2, objspace, "rgengc_rememberset_mark: mark %s\n", rb_obj_info(obj));
GC_ASSERT(RVALUE_UNCOLLECTIBLE(objspace, obj));
GC_ASSERT(RVALUE_OLD_P(objspace, obj) || RVALUE_WB_UNPROTECTED(objspace, obj));
gc_mark_children(objspace, obj);
if (RB_FL_TEST_RAW(obj, RUBY_FL_WEAK_REFERENCE)) {
rb_darray_append_without_gc(&objspace->weak_references, obj);
}
}
p += slot_size;
bitset >>= 1;
} while (bitset);
}
}
static void
rgengc_rememberset_mark(rb_objspace_t *objspace, rb_heap_t *heap)
{
size_t j;
struct heap_page *page = 0;
#if PROFILE_REMEMBERSET_MARK
int has_old = 0, has_shady = 0, has_both = 0, skip = 0;
#endif
gc_report(1, objspace, "rgengc_rememberset_mark: start\n");
ccan_list_for_each(&heap->pages, page, page_node) {
if (page->flags.has_remembered_objects | page->flags.has_uncollectible_wb_unprotected_objects) {
uintptr_t p = page->start;
short slot_size = page->slot_size;
int total_slots = page->total_slots;
int bitmap_plane_count = CEILDIV(total_slots, BITS_BITLENGTH);
bits_t bitset, bits[HEAP_PAGE_BITMAP_LIMIT];
bits_t *remembered_bits = page->remembered_bits;
bits_t *uncollectible_bits = page->uncollectible_bits;
bits_t *wb_unprotected_bits = page->wb_unprotected_bits;
#if PROFILE_REMEMBERSET_MARK
if (page->flags.has_remembered_objects && page->flags.has_uncollectible_wb_unprotected_objects) has_both++;
else if (page->flags.has_remembered_objects) has_old++;
else if (page->flags.has_uncollectible_wb_unprotected_objects) has_shady++;
#endif
/* Clear has_remembered_objects before draining the bits. A concurrent
* lock-free write barrier (another Ractor remembering a shareable object on
* this page) sets the bit first and the flag second, so clearing the flag first
* keeps the page scheduled for re-scan even if that set interleaves. The
* per-word drain is an atomic read-and-clear, so an interleaved set is not lost
* (it lands in the zeroed word). */
page->flags.has_remembered_objects = FALSE;
for (j=0; j < (size_t)bitmap_plane_count; j++) {
bits[j] = RUBY_ATOMIC_SIZE_EXCHANGE(*(volatile size_t *)&remembered_bits[j], 0)
| (uncollectible_bits[j] & wb_unprotected_bits[j]);
}
for (j=0; j < (size_t)bitmap_plane_count; j++) {
bitset = bits[j];
rgengc_rememberset_mark_plane(objspace, p, bitset, slot_size);
p += BITS_BITLENGTH * slot_size;
}
}
#if PROFILE_REMEMBERSET_MARK
else {
skip++;
}
#endif
}
#if PROFILE_REMEMBERSET_MARK
fprintf(stderr, "%d\t%d\t%d\t%d\n", has_both, has_old, has_shady, skip);
#endif
gc_report(1, objspace, "rgengc_rememberset_mark: finished\n");
}
static void
gc_bitmaps_clear(rb_objspace_t *objspace, rb_heap_t *heap, bool clear_shref)
{
struct heap_page *page = 0;
ccan_list_for_each(&heap->pages, page, page_node) {
memset(&page->mark_bits[0], 0, HEAP_PAGE_BITMAP_SIZE);
memset(&page->uncollectible_bits[0], 0, HEAP_PAGE_BITMAP_SIZE);
memset(&page->marking_bits[0], 0, HEAP_PAGE_BITMAP_SIZE);
/* A plain memset can lose a concurrent remember, but only a shareable object can
* be remembered from another Ractor's thread, and pinned_roots_mark re-marks
* those every local cycle, and this clear precedes a major that re-scans all. */
memset(&page->remembered_bits[0], 0, HEAP_PAGE_BITMAP_SIZE);
memset(&page->pinned_bits[0], 0, HEAP_PAGE_BITMAP_SIZE);
page->flags.has_uncollectible_wb_unprotected_objects = FALSE;
page->flags.has_remembered_objects = FALSE;
/* A shref is a local GC's root, so only a stop-the-world global GC may clear them:
* its unified mark re-derives them from every shareable -> unshareable edge. */
if (clear_shref) {
memset(&page->shref_bits[0], 0, HEAP_PAGE_BITMAP_SIZE);
page->flags.has_shref_objects = FALSE;
}
}
}
/* RGENGC: APIs */
NOINLINE(static void gc_writebarrier_generational(VALUE a, VALUE b, rb_objspace_t *objspace));
/* Precondition: `a` and `b` live in `objspace`. */
static void
gc_writebarrier_generational(VALUE a, VALUE b, rb_objspace_t *objspace)
{
if (RGENGC_CHECK_MODE) {
if (!RVALUE_OLD_P(objspace, a)) rb_bug("gc_writebarrier_generational: %s is not an old object.", rb_obj_info(a));
if ( RVALUE_OLD_P(objspace, b)) rb_bug("gc_writebarrier_generational: %s is an old object.", rb_obj_info(b));
if (is_incremental_marking(objspace)) rb_bug("gc_writebarrier_generational: called while incremental marking: %s -> %s", rb_obj_info(a), rb_obj_info(b));
}
/* Mark and remember a (the default behaviour).
* No lock: setting a remembered bit is atomic (rgengc_remembersetbits_set), and that is
* the only place a concurrent local GC or another Ractor's write barrier can race. */
if (!RVALUE_REMEMBERED(objspace, a)) {
rgengc_remember(objspace, a);
gc_report(1, objspace, "gc_writebarrier_generational: %s (remembered) -> %s\n", rb_obj_info(a), rb_obj_info(b));
}
check_rvalue_consistency(objspace, a);
check_rvalue_consistency(objspace, b);
}
static void
gc_mark_from(rb_objspace_t *objspace, VALUE obj, VALUE parent)
{
gc_mark_set_parent(objspace, parent);
rgengc_check_relation(objspace, obj);
if (gc_mark_set(objspace, obj) != FALSE) {
gc_aging(objspace, obj);
gc_grey(objspace, obj);
}
gc_mark_set_parent_invalid(objspace);
}
NOINLINE(static void gc_writebarrier_incremental(VALUE a, VALUE b, rb_objspace_t *objspace));
/* Precondition: `a` and `b` live in `objspace`. */
static void
gc_writebarrier_incremental(VALUE a, VALUE b, rb_objspace_t *objspace)
{
gc_report(2, objspace, "gc_writebarrier_incremental: [LG] %p -> %s\n", (void *)a, rb_obj_info(b));
if (RVALUE_BLACK_P(objspace, a)) {
if (RVALUE_WHITE_P(objspace, b)) {
if (!RVALUE_WB_UNPROTECTED(objspace, a)) {
gc_report(2, objspace, "gc_writebarrier_incremental: [IN] %p -> %s\n", (void *)a, rb_obj_info(b));
gc_mark_from(objspace, b, a);
}
}
else if (RVALUE_OLD_P(objspace, a) && !RVALUE_OLD_P(objspace, b)) {
rgengc_remember(objspace, a);
}
if (RB_UNLIKELY(objspace->flags.during_compacting)) {
MARK_IN_BITMAP(GET_HEAP_PINNED_BITS(b), b);
}
}
}
void
rb_gc_impl_writebarrier(void *objspace_ptr, VALUE a, VALUE b)
{
rb_objspace_t *objspace = objspace_ptr;
#if RGENGC_CHECK_MODE
if (SPECIAL_CONST_P(a)) rb_bug("rb_gc_writebarrier: a is special const: %"PRIxVALUE, a);
if (SPECIAL_CONST_P(b)) rb_bug("rb_gc_writebarrier: b is special const: %"PRIxVALUE, b);
#else
RBIMPL_ASSERT_OR_ASSUME(!SPECIAL_CONST_P(a));
RBIMPL_ASSERT_OR_ASSUME(!SPECIAL_CONST_P(b));
#endif
GC_ASSERT(!during_gc);
GC_ASSERT(RB_BUILTIN_TYPE(a) != T_NONE);
GC_ASSERT(RB_BUILTIN_TYPE(a) != T_MOVED);
GC_ASSERT(RB_BUILTIN_TYPE(a) != T_ZOMBIE);
/* A shareable object now references an unshareable one: record b as a shref so its
* owner's local GC roots it (the parent may live in another objspace, untraversed
* there). Only b's owner stores this, on its own page: a plain store suffices. */
if (RB_UNLIKELY(RB_FL_TEST_RAW(a, RUBY_FL_SHAREABLE)) &&
!RB_FL_TEST_RAW(b, RUBY_FL_SHAREABLE)) {
struct heap_page *bpage = GET_HEAP_PAGE(b);
if (!_MARKED_IN_BITMAP(bpage->shref_bits, bpage, b)) {
_MARK_IN_BITMAP(bpage->shref_bits, bpage, b);
bpage->flags.has_shref_objects = TRUE;
}
}
if (!is_incremental_marking(objspace)) {
/* The generational barrier covers old->young edges within one objspace only.
* NOTE: we shouldn't even check the age of `a` or `b` if they are in another
* objspace, so check locality first. */
if (gc_foreign_object_p(objspace, a) || gc_foreign_object_p(objspace, b) ||
!RVALUE_OLD_P(objspace, a) || RVALUE_OLD_P(objspace, b)) {
// do nothing
}
else {
gc_writebarrier_generational(a, b, objspace);
}
}
else {
// Shareable objects from different object spaces are kept alive by shareable bits
if (gc_foreign_object_p(objspace, a) || gc_foreign_object_p(objspace, b)) {
// do nothing
}
else {
gc_writebarrier_incremental(a, b, objspace);
}
}
}
void
rb_gc_impl_obj_became_shareable(void *objspace_ptr, VALUE obj)
{
/* An object becomes shareable on its owner thread, so this page update is
* single-writer. */
struct heap_page *page = GET_HEAP_PAGE(obj);
if (_MARKED_IN_BITMAP(page->shareable_bits, page, obj)) return;
gc_page_add_shareable(page, obj);
/* The shref bits recorded while the object was unshareable are now covered by the
* shareable pin, and a shref only points at an unshareable object. The owner thread is
* the only writer, so a plain clear is enough. */
if (_MARKED_IN_BITMAP(page->shref_bits, page, obj)) {
_CLEAR_IN_BITMAP(page->shref_bits, page, obj);
// NOTE: page->has_shref_objects could become stale here (value is true even though logically false)
}
}
void
rb_gc_impl_writebarrier_unprotect(void *objspace_ptr, VALUE obj)
{
rb_objspace_t *objspace = objspace_ptr;
/* A shareable object is never WB-unprotected. Keeping shrefs correct relies on every
* store into s->u going through the write barrier, which keeps wb_unprotected_bits
* single-writer (only the owner thread can unprotect its own unshareable objects). */
GC_ASSERT(!RB_FL_TEST_RAW(obj, RUBY_FL_SHAREABLE));
if (RVALUE_WB_UNPROTECTED(objspace, obj)) {
return;
}
else {
gc_report(2, objspace, "rb_gc_writebarrier_unprotect: %s %s\n", rb_obj_info(obj),
RVALUE_REMEMBERED(objspace, obj) ? " (already remembered)" : "");
/* No lock: per the assert obj is our own unshareable, so these bits
* (wb_unprotected, uncollectible, age) are single-writer on an owned page, and
* RVALUE_DEMOTE's remembered-bit clear is atomic against word-sharing writers. */
if (RVALUE_OLD_P(objspace, obj)) {
gc_report(1, objspace, "rb_gc_writebarrier_unprotect: %s\n", rb_obj_info(obj));
RVALUE_DEMOTE(objspace, obj);
gc_mark_set(objspace, obj);
gc_remember_unprotected(objspace, obj);
#if RGENGC_PROFILE
objspace->profile.total_shade_operation_count++;
#if RGENGC_PROFILE >= 2
objspace->profile.shade_operation_count_types[BUILTIN_TYPE(obj)]++;
#endif /* RGENGC_PROFILE >= 2 */
#endif /* RGENGC_PROFILE */
}
else {
RVALUE_AGE_RESET(obj);
}
RB_DEBUG_COUNTER_INC(obj_wb_unprotect);
MARK_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(obj), obj);
}
}
void
rb_gc_impl_copy_attributes(void *objspace_ptr, VALUE dest, VALUE obj)
{
rb_objspace_t *objspace = objspace_ptr;
if (RVALUE_WB_UNPROTECTED(objspace, obj)) {
rb_gc_impl_writebarrier_unprotect(objspace, dest);
}
rb_gc_impl_copy_finalizer(objspace, dest, obj);
}
const char *
rb_gc_impl_active_gc_name(void)
{
return "default";
}
/* NOTE: `obj` doesn't necessarily live in `objspace_ptr`, as `objspace_ptr` is just that
* of the current Ractor. */
void
rb_gc_impl_writebarrier_remember(void *objspace_ptr, VALUE obj)
{
rb_objspace_t *objspace = objspace_ptr;
// Shareable objects from other object spaces don't need to be put on the remembered set
// and are only collected during global GC, so not while incremental marking.
if (RB_LIKELY(!gc_foreign_object_p(objspace, obj))) {
gc_report(1, objspace, "rb_gc_writebarrier_remember: %s\n", rb_obj_info(obj));
if (is_incremental_marking(objspace)) {
if (RVALUE_BLACK_P(objspace, obj)) {
gc_grey(objspace, obj);
}
}
else if (RVALUE_OLD_P(objspace, obj)) {
rgengc_remember(objspace, obj);
}
}
}
struct rb_gc_object_metadata_names {
// Must be ID only
ID ID_wb_protected, ID_age, ID_old, ID_uncollectible, ID_marking,
ID_marked, ID_pinned, ID_remembered, ID_object_id, ID_shareable;
};
#define RB_GC_OBJECT_METADATA_ENTRY_COUNT (sizeof(struct rb_gc_object_metadata_names) / sizeof(ID))
static struct rb_gc_object_metadata_entry object_metadata_entries[RB_GC_OBJECT_METADATA_ENTRY_COUNT + 1];
struct rb_gc_object_metadata_entry *
rb_gc_impl_object_metadata(void *objspace_ptr, VALUE obj)
{
rb_objspace_t *objspace = objspace_ptr;
size_t n = 0;
static struct rb_gc_object_metadata_names names;
if (!names.ID_marked) {
#define I(s) names.ID_##s = rb_intern(#s)
I(wb_protected);
I(age);
I(old);
I(uncollectible);
I(marking);
I(marked);
I(pinned);
I(remembered);
I(object_id);
I(shareable);
#undef I
}
#define SET_ENTRY(na, v) do { \
GC_ASSERT(n <= RB_GC_OBJECT_METADATA_ENTRY_COUNT); \
object_metadata_entries[n].name = names.ID_##na; \
object_metadata_entries[n].val = v; \
n++; \
} while (0)
if (!RVALUE_WB_UNPROTECTED(objspace, obj)) SET_ENTRY(wb_protected, Qtrue);
SET_ENTRY(age, INT2FIX(RVALUE_AGE_GET(obj)));
if (RVALUE_OLD_P(objspace, obj)) SET_ENTRY(old, Qtrue);
if (RVALUE_UNCOLLECTIBLE(objspace, obj)) SET_ENTRY(uncollectible, Qtrue);
if (RVALUE_MARKING(objspace, obj)) SET_ENTRY(marking, Qtrue);
if (RVALUE_MARKED(objspace, obj)) SET_ENTRY(marked, Qtrue);
if (RVALUE_PINNED(objspace, obj)) SET_ENTRY(pinned, Qtrue);
if (RVALUE_REMEMBERED(objspace, obj)) SET_ENTRY(remembered, Qtrue);
if (rb_obj_id_p(obj)) SET_ENTRY(object_id, rb_obj_id(obj));
if (FL_TEST(obj, FL_SHAREABLE)) SET_ENTRY(shareable, Qtrue);
object_metadata_entries[n].name = 0;
object_metadata_entries[n].val = 0;
#undef SET_ENTRY
return object_metadata_entries;
}
void *
rb_gc_impl_ractor_cache_alloc(void *objspace_ptr, void *ractor)
{
/* No cache needed: allocation happens in a per-Ractor objspace. */
return NULL;
}
void
rb_gc_impl_ractor_cache_free(void *objspace_ptr, void *cache)
{
GC_ASSERT(cache == NULL);
}
/* The terminating Ractor's final local GC, on its own thread: roots are minimal, so the
* mark is tiny, and it reclaims what the joining side would otherwise inherit. Never
* promotes to a global GC (that would STW on every Ractor death); empty pages go
* straight back to the page pool. */
/* Finalize the zombies whose cleanup is pure C (a dfree, no Ruby-level finalizer);
* the caller has no Ruby execution context any more, so zombies with a Ruby
* finalizer stay deferred and travel to the inheritor as before. Returns whether
* anything was finalized (those pages then need one more sweep to detach). */
static bool
finalize_deferred_dfree_only(rb_objspace_t *objspace)
{
VALUE dfree_only = 0;
VALUE zombie = RUBY_ATOMIC_VALUE_EXCHANGE(heap_pages_deferred_final, 0);
while (zombie) {
rb_asan_unpoison_object(zombie, false);
VALUE next = RZOMBIE(zombie)->next;
if (FL_TEST_RAW(zombie, FL_FINALIZE)) {
/* re-defer, with the same push as rb_gc_impl_make_zombie */
VALUE prev2, next2 = heap_pages_deferred_final;
do {
RZOMBIE(zombie)->next = prev2 = next2;
next2 = RUBY_ATOMIC_VALUE_CAS(heap_pages_deferred_final, prev2, zombie);
} while (next2 != prev2);
rb_asan_poison_object(zombie);
}
else {
RZOMBIE(zombie)->next = dfree_only;
dfree_only = zombie;
}
zombie = next;
}
if (dfree_only) finalize_list(objspace, dfree_only);
bool did = dfree_only != 0;
gc_tdata_unsafe_free_publish(objspace);
return did;
}
void
rb_gc_impl_objspace_retire_gc(void *objspace_ptr)
{
rb_objspace_t *objspace = objspace_ptr;
/* The dying thread's stack is already torn down here, so the root scan must skip
* its machine context (rb_gc_mark_roots). */
objspace->flags.during_postmortem = 1;
objspace->flags.ownerless = 1;
gc_rest(objspace);
gc_start(objspace, GPR_FLAG_FULL_MARK | GPR_FLAG_IMMEDIATE_MARK | GPR_FLAG_IMMEDIATE_SWEEP);
/* The sweep above turned this heap's dead IO and the like into deferred zombies
* (the per-Ractor stdio holds a page per Ractor otherwise); finalize the C-only
* ones here and re-sweep the nearly-empty heap so their pages detach as empty. */
if (finalize_deferred_dfree_only(objspace)) {
gc_start(objspace, GPR_FLAG_FULL_MARK | GPR_FLAG_IMMEDIATE_MARK | GPR_FLAG_IMMEDIATE_SWEEP);
}
heap_pages_freeable_pages = objspace->empty_pages_count;
heap_pages_free_unused_pages(objspace);
objspace->flags.during_postmortem = 0;
}
bool
rb_gc_impl_during_postmortem_p(void *objspace_ptr)
{
rb_objspace_t *objspace = objspace_ptr;
return objspace->flags.during_postmortem != 0;
}
static void
heap_ready_to_gc(rb_objspace_t *objspace, rb_heap_t *heap)
{
if (!heap->free_pages) {
if (!heap_page_allocate_and_initialize(objspace, heap)) {
objspace->heap_pages.allocatable_bytes = HEAP_PAGE_SIZE;
heap_page_allocate_and_initialize(objspace, heap);
}
}
}
static int
ready_to_gc(rb_objspace_t *objspace)
{
if ((!objspace->flags.during_postmortem && rb_gc_gc_disabled_global_p()) || dont_gc_val() || during_gc) {
for (int i = 0; i < HEAP_COUNT; i++) {
rb_heap_t *heap = &heaps[i];
heap_ready_to_gc(objspace, heap);
}
return FALSE;
}
else {
return TRUE;
}
}
static void
gc_reset_malloc_info(rb_objspace_t *objspace, bool full_mark)
{
gc_prof_set_malloc_info(objspace);
{
int64_t inc = gc_malloc_counters_increase(objspace, &objspace->malloc_counters.counters);
gc_malloc_counters_snapshot(objspace, &objspace->malloc_counters.counters);
size_t old_limit = malloc_limit;
/* A net-negative `inc` (more freed than malloc'd since last GC) is
* treated the same as "allocated less than malloc_limit".
* This matches what we were doing pre-monotonic counters, but is it right? */
if (inc > 0 && (size_t)inc > malloc_limit) {
malloc_limit = (size_t)((size_t)inc * gc_params.malloc_limit_growth_factor);
if (malloc_limit > gc_params.malloc_limit_max) {
malloc_limit = gc_params.malloc_limit_max;
}
}
else {
malloc_limit = (size_t)(malloc_limit * 0.98); /* magic number */
if (malloc_limit < gc_params.malloc_limit_min) {
malloc_limit = gc_params.malloc_limit_min;
}
}
if (0) {
if (old_limit != malloc_limit) {
fprintf(stderr, "[%"PRIuSIZE"] malloc_limit: %"PRIuSIZE" -> %"PRIuSIZE"\n",
rb_gc_count(), old_limit, malloc_limit);
}
else {
fprintf(stderr, "[%"PRIuSIZE"] malloc_limit: not changed (%"PRIuSIZE")\n",
rb_gc_count(), malloc_limit);
}
}
}
/* reset oldmalloc info */
#if RGENGC_ESTIMATE_OLDMALLOC
MALLOC_COUNTERS_LOCK(objspace);
gc_counter_store_release(&objspace->malloc_counters.freed_in_gc, 0);
MALLOC_COUNTERS_UNLOCK(objspace);
if (full_mark) {
RUBY_ATOMIC_SIZE_EXCHANGE(objspace->malloc_counters.oldmalloc_increase, 0);
if ((objspace->profile.latest_gc_info & GPR_FLAG_MAJOR_BY_OLDMALLOC) == 0) {
objspace->rgengc.oldmalloc_increase_limit =
(size_t)(objspace->rgengc.oldmalloc_increase_limit / ((gc_params.oldmalloc_limit_growth_factor - 1)/10 + 1));
if (objspace->rgengc.oldmalloc_increase_limit < gc_params.oldmalloc_limit_min) {
objspace->rgengc.oldmalloc_increase_limit = gc_params.oldmalloc_limit_min;
}
}
}
#endif
}
/* What a collection records about itself before it runs. A global collection reports the
* driver's objspace, so it comes through here too. */
static void
gc_start_record(rb_objspace_t *objspace, unsigned int reason, bool full_mark)
{
objspace->profile.latest_gc_info = reason;
#if GC_PROFILE_MORE_DETAIL
objspace->profile.total_allocated_objects_at_gc_start = total_allocated_objects(objspace);
objspace->profile.heap_used_at_gc_start = rb_darray_size(objspace->heap_pages.sorted);
#endif
objspace->profile.weak_references_count = 0;
gc_prof_setup_new_record(objspace, reason);
gc_reset_malloc_info(objspace, full_mark);
}
static bool gc_start_global(rb_objspace_t *driver, unsigned int reason, bool compact, bool allow_skip);
/* Decide whether this collection has to be global. A local GC can reclaim neither
* shareable objects nor zombie objspaces, so once those grow past their limits only a
* global GC makes progress. All inputs belong to this objspace. */
static bool
gc_need_global_p(rb_objspace_t *objspace)
{
if (rb_gc_single_objspace_p()) return false;
/* A Ractor's death must not stop the world, so the retire GC stays local. */
if (objspace->flags.during_postmortem) return false;
if (objspace->shareable_objects > objspace->shareable_objects_limit) return true;
/* A zombie's garbage only a global cycle reclaims, but what survived the last one
* is live data, so retrigger only once TRIGGER more pages accumulate on top of it.
* Otherwise one live-heavy unjoined zombie turns every GC stop-the-world forever. */
{
size_t zp = global_objspace->zombie_total_pages;
size_t base = global_objspace->zombie_pages_survivors < zp ? global_objspace->zombie_pages_survivors : zp;
if (zp - base >= ZOMBIE_PAGES_TRIGGER) return true;
}
return false;
}
static int
garbage_collect(rb_objspace_t *objspace, unsigned int reason)
{
int ret;
#if GC_PROFILE_MORE_DETAIL
objspace->profile.prepare_time = getrusage_time();
#endif
gc_rest(objspace);
#if GC_PROFILE_MORE_DETAIL
objspace->profile.prepare_time = getrusage_time() - objspace->profile.prepare_time;
#endif
ret = gc_start(objspace, reason);
return ret;
}
static int
gc_start(rb_objspace_t *objspace, unsigned int reason)
{
unsigned int do_full_mark = !!(reason & GPR_FLAG_FULL_MARK);
if (!rb_darray_size(objspace->heap_pages.sorted)) return TRUE; /* heap is not ready */
if (!(reason & GPR_FLAG_METHOD) && !ready_to_gc(objspace)) return TRUE; /* GC is not allowed */
/* An explicit GC.start(global: true) never gets here: rb_gc_impl_start has already decided from
* the `global` keyword, and GPR_FLAG_METHOD keeps `global: false` from being promoted back. */
if (!(reason & GPR_FLAG_METHOD) && gc_need_global_p(objspace)) {
/* A global GC is always a major, so autocompact applies. */
if (gc_start_global(objspace, reason, ruby_enable_autocompact, true)) {
return TRUE;
}
/* Fall through to a local GC */
}
rb_gc_initialize_vm_context(&objspace->vm_context);
GC_ASSERT(gc_mode(objspace) == gc_mode_none, "gc_mode is %s\n", gc_mode_name(gc_mode(objspace)));
GC_ASSERT(!is_lazy_sweeping(objspace));
GC_ASSERT(!is_incremental_marking(objspace));
/* reason may be clobbered, later, so keep set immediate_sweep here */
objspace->flags.immediate_sweep = !!(reason & GPR_FLAG_IMMEDIATE_SWEEP);
if (ruby_gc_stressful) {
int flag = FIXNUM_P(ruby_gc_stress_mode) ? FIX2INT(ruby_gc_stress_mode) : 0;
if ((flag & (1 << gc_stress_no_major)) == 0) {
do_full_mark = TRUE;
}
objspace->flags.immediate_sweep = !(flag & (1<<gc_stress_no_immediate_sweep));
}
if (gc_needs_major_flags) {
reason |= gc_needs_major_flags;
do_full_mark = TRUE;
}
/* if major gc has been disabled, never do a full mark */
if (!gc_config_full_mark_val) {
do_full_mark = FALSE;
}
gc_needs_major_flags = GPR_FLAG_NONE;
if (do_full_mark && (reason & GPR_FLAG_MAJOR_MASK) == 0) {
reason |= GPR_FLAG_MAJOR_BY_FORCE; /* GC by CAPI, METHOD, and so on. */
}
if (objspace->flags.dont_incremental ||
reason & GPR_FLAG_IMMEDIATE_MARK ||
ruby_gc_stressful) {
gc_set_incremental_marking(objspace, FALSE);
}
else {
gc_set_incremental_marking(objspace, do_full_mark);
}
/* Compaction on the local GC path (autocompact) runs only with a single objspace:
* without the stop-the-world barrier, moving objects would break cross-objspace
* references. With multiple objspaces GC.compact and autocompact go through the
* compacting global GC instead (rb_gc_impl_start, or the promotion above). */
if (do_full_mark && ruby_enable_autocompact && rb_gc_single_objspace_p()) {
objspace->flags.during_compacting = TRUE;
#if RGENGC_CHECK_MODE
objspace->rcompactor.compare_func = ruby_autocompact_compare_func;
#endif
}
else {
objspace->flags.during_compacting = !!(reason & GPR_FLAG_COMPACT);
if (objspace->flags.during_compacting && !rb_gc_single_objspace_p()) {
// compaction is currently global GC only with more than 1 running Ractor
objspace->flags.during_compacting = FALSE;
}
}
if (!GC_ENABLE_LAZY_SWEEP || objspace->flags.dont_incremental) {
objspace->flags.immediate_sweep = TRUE;
}
if (objspace->flags.immediate_sweep) reason |= GPR_FLAG_IMMEDIATE_SWEEP;
/* Enter after during_compacting is decided */
unsigned int lock_lev;
gc_enter(objspace, gc_enter_event_start, &lock_lev);
gc_report(1, objspace, "gc_start(reason: %x) => %u, %d, %d\n",
reason,
do_full_mark, !is_incremental_marking(objspace), objspace->flags.immediate_sweep);
RB_DEBUG_COUNTER_INC(gc_count);
if (reason & GPR_FLAG_MAJOR_MASK) {
(void)RB_DEBUG_COUNTER_INC_IF(gc_major_nofree, reason & GPR_FLAG_MAJOR_BY_NOFREE);
(void)RB_DEBUG_COUNTER_INC_IF(gc_major_oldgen, reason & GPR_FLAG_MAJOR_BY_OLDGEN);
(void)RB_DEBUG_COUNTER_INC_IF(gc_major_shady, reason & GPR_FLAG_MAJOR_BY_SHADY);
(void)RB_DEBUG_COUNTER_INC_IF(gc_major_force, reason & GPR_FLAG_MAJOR_BY_FORCE);
#if RGENGC_ESTIMATE_OLDMALLOC
(void)RB_DEBUG_COUNTER_INC_IF(gc_major_oldmalloc, reason & GPR_FLAG_MAJOR_BY_OLDMALLOC);
#endif
}
else {
(void)RB_DEBUG_COUNTER_INC_IF(gc_minor_newobj, reason & GPR_FLAG_NEWOBJ);
(void)RB_DEBUG_COUNTER_INC_IF(gc_minor_malloc, reason & GPR_FLAG_MALLOC);
(void)RB_DEBUG_COUNTER_INC_IF(gc_minor_method, reason & GPR_FLAG_METHOD);
(void)RB_DEBUG_COUNTER_INC_IF(gc_minor_capi, reason & GPR_FLAG_CAPI);
(void)RB_DEBUG_COUNTER_INC_IF(gc_minor_stress, reason & GPR_FLAG_STRESS);
}
objspace->profile.count++;
gc_start_record(objspace, reason, do_full_mark);
gc_event_hook(objspace, RUBY_INTERNAL_EVENT_GC_START);
GC_ASSERT(during_gc);
gc_prof_timer_start(objspace);
{
if (gc_marks(objspace, do_full_mark)) {
gc_sweep(objspace);
}
}
gc_prof_timer_stop(objspace);
gc_exit(objspace, gc_enter_event_start, &lock_lev);
/* Verify after the GC, at a real safepoint with during_gc cleared: mid-GC
* gc_verify_internal_consistency() takes the VM lock and a barrier, which would join
* another Ractor's global GC barrier and let it collect on this half-collected heap. */
#if RGENGC_CHECK_MODE >= 2
gc_verify_internal_consistency(objspace);
#endif
return TRUE;
}
static void
gc_rest(rb_objspace_t *objspace)
{
if (is_incremental_marking(objspace) || is_lazy_sweeping(objspace)) {
unsigned int lock_lev;
gc_enter(objspace, gc_enter_event_rest, &lock_lev);
if (is_incremental_marking(objspace)) {
gc_marking_enter(objspace);
gc_marks_rest(objspace);
gc_marking_exit(objspace);
gc_sweep(objspace);
}
if (is_lazy_sweeping(objspace)) {
gc_sweeping_enter(objspace);
gc_sweep_rest(objspace);
gc_sweeping_exit(objspace);
}
gc_exit(objspace, gc_enter_event_rest, &lock_lev);
if (RGENGC_CHECK_MODE >= 2) gc_verify_internal_consistency(objspace); /* after GC, see gc_start */
}
}
struct objspace_and_reason {
rb_objspace_t *objspace;
unsigned int reason;
};
static void
gc_current_status_fill(rb_objspace_t *objspace, char *buff)
{
int i = 0;
if (is_marking(objspace)) {
buff[i++] = 'M';
if (is_full_marking(objspace)) buff[i++] = 'F';
if (is_incremental_marking(objspace)) buff[i++] = 'I';
}
else if (is_sweeping(objspace)) {
buff[i++] = 'S';
if (is_lazy_sweeping(objspace)) buff[i++] = 'L';
}
else {
buff[i++] = 'N';
}
buff[i] = '\0';
}
static const char *
gc_current_status(rb_objspace_t *objspace)
{
static char buff[0x10];
gc_current_status_fill(objspace, buff);
return buff;
}
#if PRINT_ENTER_EXIT_TICK
static tick_t last_exit_tick;
static tick_t enter_tick;
static int enter_count = 0;
static char last_gc_status[0x10];
static inline void
gc_record(rb_objspace_t *objspace, int direction, const char *event)
{
if (direction == 0) { /* enter */
enter_count++;
enter_tick = tick();
gc_current_status_fill(objspace, last_gc_status);
}
else { /* exit */
tick_t exit_tick = tick();
char current_gc_status[0x10];
gc_current_status_fill(objspace, current_gc_status);
#if 1
/* [last mutator time] [gc time] [event] */
fprintf(stderr, "%"PRItick"\t%"PRItick"\t%s\t[%s->%s|%c]\n",
enter_tick - last_exit_tick,
exit_tick - enter_tick,
event,
last_gc_status, current_gc_status,
(objspace->profile.latest_gc_info & GPR_FLAG_MAJOR_MASK) ? '+' : '-');
last_exit_tick = exit_tick;
#else
/* [enter_tick] [gc time] [event] */
fprintf(stderr, "%"PRItick"\t%"PRItick"\t%s\t[%s->%s|%c]\n",
enter_tick,
exit_tick - enter_tick,
event,
last_gc_status, current_gc_status,
(objspace->profile.latest_gc_info & GPR_FLAG_MAJOR_MASK) ? '+' : '-');
#endif
}
}
#else /* PRINT_ENTER_EXIT_TICK */
static inline void
gc_record(rb_objspace_t *objspace, int direction, const char *event)
{
/* null */
}
#endif /* PRINT_ENTER_EXIT_TICK */
static const char *
gc_enter_event_cstr(enum gc_enter_event event)
{
switch (event) {
case gc_enter_event_start: return "start";
case gc_enter_event_continue: return "continue";
case gc_enter_event_rest: return "rest";
case gc_enter_event_finalizer: return "finalizer";
case gc_enter_event_global: return "global";
case gc_enter_event_global_auto: return "global_auto";
}
return NULL;
}
static void
gc_enter_count(enum gc_enter_event event)
{
switch (event) {
case gc_enter_event_start: RB_DEBUG_COUNTER_INC(gc_enter_start); break;
case gc_enter_event_continue: RB_DEBUG_COUNTER_INC(gc_enter_continue); break;
case gc_enter_event_rest: RB_DEBUG_COUNTER_INC(gc_enter_rest); break;
case gc_enter_event_finalizer: RB_DEBUG_COUNTER_INC(gc_enter_finalizer); break;
case gc_enter_event_global: RB_DEBUG_COUNTER_INC(gc_enter_start); break;
case gc_enter_event_global_auto: RB_DEBUG_COUNTER_INC(gc_enter_start); break;
}
}
static bool current_process_time(struct timespec *ts);
/* A gc phase must be timed on the collecting thread's own cpu. A local gc runs
* while the other ractors keep going, and process cpu time counts their work as
* gc: with eight busy ractors the same ten collections were reported as 131ms
* instead of 3ms, more than the wall clock they ran in. The kernel also answers
* this one without walking every thread in the process. */
static bool
current_thread_time(struct timespec *ts)
{
#if defined(HAVE_CLOCK_GETTIME) && defined(CLOCK_THREAD_CPUTIME_ID)
{
static int try_clock_gettime = 1;
if (try_clock_gettime) {
if (clock_gettime(CLOCK_THREAD_CPUTIME_ID, ts) == 0) {
return true;
}
else {
try_clock_gettime = 0;
}
}
}
#endif
return current_process_time(ts);
}
static void
gc_clock_start(struct timespec *ts)
{
if (!current_thread_time(ts)) {
ts->tv_sec = 0;
ts->tv_nsec = 0;
}
}
static unsigned long long
gc_clock_end(struct timespec *ts)
{
struct timespec end_time;
if ((ts->tv_sec > 0 || ts->tv_nsec > 0) &&
current_thread_time(&end_time) &&
end_time.tv_sec >= ts->tv_sec) {
return (unsigned long long)(end_time.tv_sec - ts->tv_sec) * (1000 * 1000 * 1000) +
(end_time.tv_nsec - ts->tv_nsec);
}
return 0;
}
static void
gc_process_stat_after_fork_i(void *objspace_ptr, void *data)
{
rb_objspace_t *objspace = objspace_ptr;
rb_native_mutex_initialize(&objspace->process_stat.lock);
}
static inline bool
gc_local_gc_holds_vm_lock(void)
{
return rb_gc_single_objspace_p();
}
static inline bool
gc_enter(rb_objspace_t *objspace, enum gc_enter_event event, unsigned int *lock_lev)
{
/*
* NOTE: The GC must never take the barrier VM lock from inside itself: the waiter could
* join a pending barrier mid-collection and expose its half-collected heap to the global
* GC. A no-barrier lock is safe. Other shared structures the GC paths touch use their own
* native mutexes or the page-pool lock. */
*lock_lev = 0;
RUBY_DTRACE_GC_HOOK(ENTER, event);
if (objspace->profile.run) {
switch (event) {
case gc_enter_event_start:
case gc_enter_event_continue:
case gc_enter_event_rest:
case gc_enter_event_global:
case gc_enter_event_global_auto:
/* A global GC is the longest pause the process takes, so it is the last thing
* the profiler may leave unmeasured. The switch below stops the world for it,
* which is exactly the interval gc_stop_time is meant to name, so start the
* clock here like a local collection does. */
objspace->profile.gc_pause_start_time = rb_hrtime_now();
break;
case gc_enter_event_finalizer:
break;
}
}
switch (event) {
case gc_enter_event_global:
*lock_lev = RB_GC_VM_LOCK();
// stop other ractors
rb_gc_vm_barrier();
break;
case gc_enter_event_global_auto:
*lock_lev = RB_GC_VM_LOCK();
if (!gc_need_global_p(objspace)) {
RB_GC_VM_UNLOCK(*lock_lev);
*lock_lev = 0;
objspace->profile.gc_pause_start_time = 0;
return false;
}
rb_gc_vm_barrier();
break;
case gc_enter_event_finalizer:
/* Shutdown finalizers read VM-global tables (fstring, symbol) and free T_DATA that
* is not thread-safe, so take the no-barrier VM lock. */
*lock_lev = RB_GC_VM_LOCK_NO_BARRIER();
break;
default:
objspace->flags.gc_lock_barrier = FALSE;
if (objspace->flags.during_compacting) {
/* Compaction relocates objects and rewrites every Ractor's JIT and global
* references, so it stops the world with a barrier VM lock. rb_gc_vm_barrier is
* a reentrant no-op with a single Ractor, so an inner barrier request during the
* move folds into this one and gc_exit ends it. */
*lock_lev = RB_GC_VM_LOCK();
rb_gc_vm_barrier();
objspace->flags.gc_lock_barrier = TRUE;
}
else if (gc_local_gc_holds_vm_lock()) {
*lock_lev = RB_GC_VM_LOCK_NO_BARRIER();
}
break;
}
if (objspace->profile.gc_pause_start_time) {
objspace->profile.gc_stw_start_time = rb_hrtime_now();
objspace->profile.gc_stop_time = rb_hrtime_sub(
objspace->profile.gc_stw_start_time,
objspace->profile.gc_pause_start_time);
}
gc_enter_count(event);
if (RB_UNLIKELY(during_gc != 0)) rb_bug("during_gc != 0");
if (RGENGC_CHECK_MODE >= 3) gc_verify_internal_consistency(objspace);
during_gc = TRUE;
RUBY_DEBUG_LOG("%s (%s)",gc_enter_event_cstr(event), gc_current_status(objspace));
gc_report(1, objspace, "gc_enter: %s [%s]\n", gc_enter_event_cstr(event), gc_current_status(objspace));
gc_record(objspace, 0, gc_enter_event_cstr(event));
gc_event_hook(objspace, RUBY_INTERNAL_EVENT_GC_ENTER);
return true;
}
static inline void
gc_exit(rb_objspace_t *objspace, enum gc_enter_event event, unsigned int *lock_lev)
{
GC_ASSERT(during_gc != 0);
RUBY_DTRACE_GC_HOOK(EXIT, event);
gc_event_hook(objspace, RUBY_INTERNAL_EVENT_GC_EXIT);
if (objspace->profile.gc_pause_start_time) {
if (gc_prof_enabled(objspace)) {
rb_hrtime_t now = rb_hrtime_now();
gc_profile_record *record = gc_prof_record(objspace);
record->gc_pause_time = rb_hrtime_add(record->gc_pause_time,
rb_hrtime_sub(now, objspace->profile.gc_pause_start_time));
record->gc_stop_time = rb_hrtime_add(record->gc_stop_time,
objspace->profile.gc_stop_time);
record->gc_stw_time = rb_hrtime_add(record->gc_stw_time,
rb_hrtime_sub(now, objspace->profile.gc_stw_start_time));
}
objspace->profile.gc_pause_start_time = 0;
objspace->profile.gc_stw_start_time = 0;
objspace->profile.gc_stop_time = 0;
}
gc_record(objspace, 1, gc_enter_event_cstr(event));
RUBY_DEBUG_LOG("%s (%s)", gc_enter_event_cstr(event), gc_current_status(objspace));
gc_report(1, objspace, "gc_exit: %s [%s]\n", gc_enter_event_cstr(event), gc_current_status(objspace));
during_gc = FALSE;
gc_process_stat_publish(objspace);
switch (event) {
case gc_enter_event_global:
case gc_enter_event_global_auto:
RB_GC_VM_UNLOCK(*lock_lev);
break;
case gc_enter_event_finalizer:
RB_GC_VM_UNLOCK_NO_BARRIER(*lock_lev);
break;
default:
if (*lock_lev != 0) {
if (objspace->flags.gc_lock_barrier) {
objspace->flags.gc_lock_barrier = FALSE;
RB_GC_VM_UNLOCK(*lock_lev);
}
else {
RB_GC_VM_UNLOCK_NO_BARRIER(*lock_lev);
}
}
break;
}
}
#ifndef MEASURE_GC
#define MEASURE_GC (objspace->flags.measure_gc)
#endif
static void
gc_marking_enter(rb_objspace_t *objspace)
{
GC_ASSERT(during_gc != 0);
gc_prof_mark_timer_start(objspace);
if (gc_prof_enabled(objspace)) {
objspace->profile.gc_mark_phase_wall_start_time = rb_hrtime_now();
}
if (MEASURE_GC) {
gc_clock_start(&objspace->profile.marking_start_time);
}
rb_gc_initialize_vm_context(&objspace->vm_context);
}
static void
gc_marking_exit(rb_objspace_t *objspace)
{
GC_ASSERT(during_gc != 0);
if (MEASURE_GC) {
objspace->profile.marking_time_ns += gc_clock_end(&objspace->profile.marking_start_time);
}
if (gc_prof_enabled(objspace)) {
gc_profile_record *record = gc_prof_record(objspace);
record->gc_mark_wall_time = rb_hrtime_add(record->gc_mark_wall_time,
elapsed_hrtime_from(objspace->profile.gc_mark_phase_wall_start_time));
}
gc_prof_mark_timer_stop(objspace);
}
static void
gc_sweeping_cpu_enter(rb_objspace_t *objspace)
{
if (MEASURE_GC) {
gc_clock_start(&objspace->profile.sweeping_start_time);
}
}
static void
gc_sweeping_cpu_exit(rb_objspace_t *objspace)
{
if (MEASURE_GC) {
objspace->profile.sweeping_time_ns += gc_clock_end(&objspace->profile.sweeping_start_time);
}
}
static void
gc_sweeping_enter(rb_objspace_t *objspace)
{
GC_ASSERT(during_gc != 0);
if (gc_prof_enabled(objspace)) {
objspace->profile.gc_sweep_phase_wall_start_time = rb_hrtime_now();
objspace->profile.gc_sweep_excluded_wall_time = 0;
}
gc_sweeping_cpu_enter(objspace);
rb_gc_initialize_vm_context(&objspace->vm_context);
}
static void
gc_sweeping_exit(rb_objspace_t *objspace)
{
GC_ASSERT(during_gc != 0);
gc_sweeping_cpu_exit(objspace);
if (gc_prof_enabled(objspace)) {
rb_hrtime_t sweep_wall_time = elapsed_hrtime_from(objspace->profile.gc_sweep_phase_wall_start_time);
gc_profile_record *record = gc_prof_record(objspace);
sweep_wall_time = rb_hrtime_sub(sweep_wall_time,
objspace->profile.gc_sweep_excluded_wall_time);
record->gc_sweep_wall_time = rb_hrtime_add(record->gc_sweep_wall_time,
sweep_wall_time);
objspace->profile.gc_sweep_excluded_wall_time = 0;
}
}
static void *
gc_with_gvl(void *ptr)
{
struct objspace_and_reason *oar = (struct objspace_and_reason *)ptr;
return (void *)(VALUE)garbage_collect(oar->objspace, oar->reason);
}
int ruby_thread_has_gvl_p(void);
static int
garbage_collect_with_gvl(rb_objspace_t *objspace, unsigned int reason)
{
if (rb_gc_gc_disabled_global_p() || dont_gc_val()) {
return TRUE;
}
else if (!ruby_native_thread_p()) {
return TRUE;
}
else if (!ruby_thread_has_gvl_p()) {
void *ret;
struct objspace_and_reason oar;
oar.objspace = objspace;
oar.reason = reason;
ret = rb_thread_call_with_gvl(gc_with_gvl, (void *)&oar);
return !!ret;
}
else {
return garbage_collect(objspace, reason);
}
}
static int
gc_set_candidate_object_i(void *vstart, void *vend, size_t stride, void *data)
{
rb_objspace_t *objspace = (rb_objspace_t *)data;
VALUE v = (VALUE)vstart;
for (; v != (VALUE)vend; v += stride) {
asan_unpoisoning_object(v) {
switch (BUILTIN_TYPE(v)) {
case T_NONE:
case T_ZOMBIE:
break;
default:
rb_gc_prepare_heap_process_object(v);
if (!RVALUE_OLD_P(objspace, v) && !RVALUE_WB_UNPROTECTED(objspace, v)) {
RVALUE_AGE_SET_CANDIDATE(objspace, v);
}
}
}
}
return 0;
}
bool
rb_gc_impl_multi_objspace_p(void)
{
return true;
}
bool
rb_gc_impl_during_global_gc_p(void *objspace_ptr)
{
rb_objspace_t *objspace = objspace_ptr;
return objspace->flags.during_global_gc != 0;
}
bool
rb_gc_impl_obj_foreign_p(void *objspace_ptr, VALUE obj)
{
return gc_foreign_object_p(objspace_ptr, obj);
}
/* Whether obj is recorded as an unshareable object referenced from a shareable one. For
* the verifier: a shareable -> unshareable edge is only accepted if the write barrier
* recorded it here. */
bool
rb_gc_impl_shref_marked_p(void *objspace_ptr, VALUE obj)
{
return MARKED_IN_BITMAP(GET_HEAP_SHREF_BITS(obj), obj) != 0;
}
/* What a zombie objspace is worth to the zombie page trigger, floored: the retire GC
* can empty an ownerless objspace outright, yet its struct, mark stack and heap_page
* structs stay in malloc and it keeps the world multi-objspace until a global GC
* merges it, so a page-less zombie must still push the trigger. Never 0, so a
* nonzero zombie_pages also marks the objspace as a zombie. */
static size_t
objspace_zombie_pages(rb_objspace_t *objspace)
{
size_t pages = rb_darray_size(objspace->heap_pages.sorted);
return pages > HEAP_COUNT ? pages : HEAP_COUNT;
}
/* gc.c parked the objspace in its zombie table, under the VM lock. */
void
rb_gc_impl_objspace_zombify(void *objspace_ptr)
{
rb_objspace_t *objspace = objspace_ptr;
GC_ASSERT(objspace->zombie_pages == 0);
objspace->zombie_pages = objspace_zombie_pages(objspace);
global_objspace->zombie_total_pages += objspace->zombie_pages;
}
/* Re-measure every zombie this cycle covered, by difference, so a zombie parked after
* the snapshot keeps the pages it was parked with. Inside the barrier. */
static void
gc_global_refresh_zombie_pages(void)
{
for (size_t i = 0; i < global_objspace->global_gc.n_objspaces; i++) {
rb_objspace_t *objspace = global_objspace->global_gc.objspaces[i];
if (objspace->zombie_pages == 0) continue;
size_t pages = objspace_zombie_pages(objspace);
global_objspace->zombie_total_pages -= objspace->zombie_pages;
global_objspace->zombie_total_pages += pages;
objspace->zombie_pages = pages;
}
global_objspace->zombie_pages_survivors = global_objspace->zombie_total_pages;
}
static void
gc_global_objspaces_i(void *os, void *data)
{
if (global_objspace->global_gc.n_objspaces == global_objspace->global_gc.objspaces_capa) {
size_t new_capa = global_objspace->global_gc.objspaces_capa ? global_objspace->global_gc.objspaces_capa * 2 : 16;
struct rb_objspace **new_list = realloc(global_objspace->global_gc.objspaces, new_capa * sizeof(*new_list));
if (new_list == NULL) rb_bug("gc_global_objspaces_i: realloc failed");
global_objspace->global_gc.objspaces = new_list;
global_objspace->global_gc.objspaces_capa = new_capa;
}
global_objspace->global_gc.objspaces[global_objspace->global_gc.n_objspaces++] = os;
}
/* Re-snapshot every objspace this cycle covers, zombies included. The objspaces/capa
* buffer is reused from the previous cycle. */
static void
gc_global_snapshot_objspaces(void)
{
global_objspace->global_gc.n_objspaces = 0;
rb_gc_vm_each_objspace(gc_global_objspaces_i, NULL);
#if RGENGC_CHECK_MODE
/* Check that the incrementally maintained page_index agrees with the per-objspace
* sorted arrays. */
size_t total = 0, zombie_total = 0;
for (size_t i = 0; i < global_objspace->global_gc.n_objspaces; i++) {
total += rb_darray_size(global_objspace->global_gc.objspaces[i]->heap_pages.sorted);
zombie_total += global_objspace->global_gc.objspaces[i]->zombie_pages;
}
GC_ASSERT(total == global_objspace->page_index.n_pages);
/* Every zombie is enumerated, so the snapshot sees all of zombie_total_pages. */
GC_ASSERT(zombie_total == global_objspace->zombie_total_pages);
#endif
}
/* Global GC: stop every Ractor and clear/mark/sweep all objspaces as one heap. It is the
* only collector that can free shareable objects and decide cross-objspace reachability
* precisely. */
/* The global GC's generic_fields weak pass, after the unified mark fixpoint, before the
* sweep. Per-object rb_mark_generic_ivar is a no-op during a global GC (the driver has
* GET_RACTOR() != owner); the whole table is swept here instead. Weak-KEY: mark the val
* (fields_obj, a strong child) only for a live key, drain dead keys' entries. Marking a
* val can make another key live, so repeat to a fixpoint. */
struct genfields_mark_arg {
rb_objspace_t *objspace;
bool progress;
};
static int
genfields_mark_i(VALUE key, VALUE val, void *arg)
{
struct genfields_mark_arg *a = (struct genfields_mark_arg *)arg;
if (RB_SPECIAL_CONST_P(val) || !RVALUE_MARKED_BITMAP(key)) {
return ST_CONTINUE;
}
/* Record the old(key)->young(val) edge with the host (key) as parent, even when val
* is already marked: a conservative machine-stack scan can mark a fresh fields_obj
* parentless before this pass, and branching on the mark bit would leave the key
* unremembered, so the next minor GC misses the young val ("WB miss (O->Y)").
* gc_mark runs rgengc_check_relation before its already-marked return: call always. */
bool newly = !RVALUE_MARKED_BITMAP(val);
gc_mark_set_parent(a->objspace, key);
gc_mark(a->objspace, val);
if (newly) a->progress = true;
return ST_CONTINUE;
}
static bool
genfields_dead_p(VALUE key)
{
return RVALUE_MARKED_BITMAP(key) == 0;
}
static void
gc_global_mark_generic_fields(rb_objspace_t *driver)
{
struct genfields_mark_arg arg = { driver, false };
do {
arg.progress = false;
/* Each entry's mark sets parent=key (genfields_mark_i) so the generational WB is
* recorded correctly. gc_mark_stacked_objects_all sets its own per-object parent,
* so restore the invalid parent (the poison contract) before calling it. */
rb_gc_vm_generic_fields_mark_foreach(genfields_mark_i, &arg);
gc_mark_set_parent_invalid(driver);
if (arg.progress) {
gc_mark_stacked_objects_all(driver);
}
} while (arg.progress);
rb_gc_vm_generic_fields_drain_dead(genfields_dead_p);
}
/* Two Ractors choosing a global GC at once are serialized by the VM lock in gc_enter. If two
* globals start concurrently, only one global will run and the other will run a local GC after
* the barrier ends. */
static bool
gc_start_global(rb_objspace_t *driver, unsigned int reason, bool compact, bool allow_skip)
{
unsigned int lock_lev;
enum gc_enter_event event = allow_skip ? gc_enter_event_global_auto : gc_enter_event_global;
if (!gc_enter(driver, event, &lock_lev)) {
return false;
}
reason |= GPR_FLAG_GLOBAL;
/* A global GC is a collection of the driver's objspace too, so report it like a local
* one: it records a profile entry and reports what triggered it. */
gc_start_record(driver, reason, true);
gc_event_hook(driver, RUBY_INTERNAL_EVENT_GC_START);
gc_prof_timer_start(driver);
gc_global_snapshot_objspaces();
/* Mark every objspace as in a global GC before step 3 settles the lazy sweeps: the
* settle frees other objspaces' garbage on the driver thread, and
* rb_free_generic_ivar must see "global GC in progress" to defer generic_fields
* removal to the weak-pass drain. */
for (size_t i = 0; i < global_objspace->global_gc.n_objspaces; i++) {
global_objspace->global_gc.objspaces[i]->flags.during_global_gc = TRUE;
}
/* A global GC collects every objspace, so each needs the malloc-counter reset the
* driver got in gc_start_record; without it, gc_sweep_finish advancing free_at_last_gc
* (step 9) would leave their malloc_increase overstated by everything swept here. */
for (size_t i = 0; i < global_objspace->global_gc.n_objspaces; i++) {
rb_objspace_t *const os = global_objspace->global_gc.objspaces[i];
if (os != driver) {
os->profile.latest_gc_info = reason;
gc_reset_malloc_info(os, true);
}
}
/* step 3: settle every lazy sweep so the mark bits' meaning is fixed before the clear
* below. (during_gc is a macro over the local "objspace".) rb_gc_get_ec() resolves
* through objspace->vm_context during a GC, so initialize it for all: the driver
* thread runs every objspace's phases. */
for (size_t i = 0; i < global_objspace->global_gc.n_objspaces; i++) {
rb_objspace_t *objspace = global_objspace->global_gc.objspaces[i];
/* The barrier can stop a Ractor between the steps of its own incremental mark, and
* nothing else finishes another objspace's mark. Drop the partial mark rather
* than clear its flags in step 5 under a live gray stack: the owner would resume
* with a mark stack into a heap this GC has since re-marked and swept. Nothing is
* lost, the unified mark below redoes the work. */
if (is_incremental_marking(objspace)) {
gc_abort_incremental_marking(objspace);
}
GC_ASSERT(!is_incremental_marking(objspace));
GC_ASSERT(is_mark_stack_empty(&objspace->mark_stack));
rb_gc_initialize_vm_context(&objspace->vm_context);
if (objspace != driver) during_gc = TRUE;
gc_sweep_rest(objspace);
}
/* step 5: clear every objspace's mark bits, remembered sets, generation counters and
* shrefs (missing even one leaves a stale mark bit and a UAF). (heaps is a macro over
* the local "objspace".) */
for (size_t i = 0; i < global_objspace->global_gc.n_objspaces; i++) {
rb_objspace_t *objspace = global_objspace->global_gc.objspaces[i];
objspace->flags.during_minor_gc = FALSE;
gc_set_incremental_marking(objspace, FALSE);
/* The unified mark is precise and does not pin, so the per-objspace sweep below must
* not re-check against a stale local cycle. */
objspace->last_cycle_pinned = 0;
objspace->rgengc.uncollectible_wb_unprotected_objects = 0;
objspace->rgengc.old_objects = 0;
objspace->profile.count++;
objspace->profile.global_gc_count++;
objspace->rgengc.last_major_gc = objspace->profile.count;
objspace->rgengc.need_major_gc = GPR_FLAG_NONE;
objspace->marked_slots = 0;
for (int h = 0; h < HEAP_COUNT; h++) {
rb_heap_t *heap = &heaps[h];
gc_bitmaps_clear(objspace, heap, true);
heap_move_pooled_pages_to_free_pages(heap);
}
}
driver->profile.global_gc_invocation_count++;
global_objspace->global_gc.count++;
/* Enable compaction in every objspace before the mark: the unified conservative root
* scan then pins machine-stack referents (gc_pin only pins while during_compacting)
* and step 9's sweep relocates the rest. global_gc.compacting defers the
* reference-update phase to phase 2 below (two phases, safe across objspaces). */
global_objspace->global_gc.compacting = compact;
if (compact) {
for (size_t i = 0; i < global_objspace->global_gc.n_objspaces; i++) {
rb_objspace_t *objspace = global_objspace->global_gc.objspaces[i];
objspace->flags.during_compacting = TRUE;
#if RGENGC_CHECK_MODE
if (ruby_enable_autocompact) {
objspace->rcompactor.compare_func = ruby_autocompact_compare_func;
}
#endif
/* A global GC skips gc_marks_start, which is what resets pinned_slots for a
* compacting local GC, so reset it here. step 5 cleared pinned_bits; the
* conservative mark re-pins machine-stack referents. */
for (int h = 0; h < HEAP_COUNT; h++) {
struct heap_page *page = NULL;
ccan_list_for_each(&heaps[h].pages, page, page_node) {
page->pinned_slots = 0;
}
}
}
}
/* steps 6-7: every Ractor's roots (gc.c walks them all), then one unified precise
* mark. A global GC does not go through gc_marks, so the marking
* phase is opened here instead; it closes after rb_ractor_finish_marking below, which is
* where gc_marks_finish ends for a local collection. */
gc_marking_enter(driver);
mark_roots(driver, NULL);
gc_mark_stacked_objects_all(driver);
/* Run the generic_fields weak pass after the mark fixpoint: mark the vals (fields_obj)
* of live keys and drain the entries of dead ones. The per-object rb_mark_generic_ivar
* is a no-op during a global GC, so this is the only path that marks generic_fields. */
gc_global_mark_generic_fields(driver);
gc_event_hook(driver, RUBY_INTERNAL_EVENT_GC_END_MARK);
/* step 8 */
gc_update_weak_references(driver);
/* This cycle's root pass over every Ractor has swept the deleted ractor-local keys out of
* each storage. Free the key structs while still inside the barrier (a local GC never
* can; see rb_ractor_finish_marking). */
rb_ractor_finish_marking(true);
gc_marking_exit(driver);
/* step 9: sweep every objspace inside the barrier, not lazily. Dead shareable objects
* are reclaimed here and emptied pages go back to the pool. */
if (!compact) {
for (size_t i = 0; i < global_objspace->global_gc.n_objspaces; i++) {
rb_objspace_t *os = global_objspace->global_gc.objspaces[i];
unsigned int prev_immediate = os->flags.immediate_sweep;
os->flags.immediate_sweep = TRUE;
gc_sweep(os);
os->flags.immediate_sweep = prev_immediate;
}
}
else {
/* The move -> update-references -> free flow runs as three passes across ALL
* objspaces, not per objspace: (a) updating references must see every objspace's
* forwarding (a reference can point at a moved foreign object), and (b) freeing
* source pages must wait until everyone is updated (or another objspace's update
* reads a freed T_MOVED). The read barrier is installed once for all passes. */
install_handlers();
/* Only the driver records a profile entry for a global GC (gc_start_record), so time
* only the driver's compaction work. The move/update/free below runs inside the
* driver's sweep phase (gc_sweeping_enter/exit); attribute it to GC_COMPACT_WALL_TIME
* and exclude it from the driver's sweep wall time so the two do not double-count,
* mirroring the compacting branch of the local gc_sweep(). */
const bool driver_prof = gc_prof_enabled(driver);
rb_hrtime_t driver_compact_wall_time = 0;
/* pass 1 (move): relocate every objspace and leave T_MOVED forwarding behind. */
for (size_t i = 0; i < global_objspace->global_gc.n_objspaces; i++) {
rb_objspace_t *os = global_objspace->global_gc.objspaces[i];
gc_sweeping_enter(os);
gc_sweep_start(os); /* mode -> sweeping, order the heap for compaction */
rb_hrtime_t t0 = (os == driver && driver_prof) ? rb_hrtime_now() : 0;
gc_compact_relocate(os); /* mode -> compacting, move */
if (os == driver && driver_prof) {
driver_compact_wall_time = rb_hrtime_add(driver_compact_wall_time, elapsed_hrtime_from(t0));
}
gc_sweeping_cpu_exit(os);
}
/* pass 2 (update): all forwarding now exists, so update every objspace's
* references (cross-objspace ones resolve too); gc_compact_finish also unprotects
* pages and clears during_compacting. The move-or-mark decision reads
* rb_gc_get_objspace()'s during_reference_updating: set it on every objspace. */
gc_sweeping_cpu_enter(driver);
for (size_t i = 0; i < global_objspace->global_gc.n_objspaces; i++) {
global_objspace->global_gc.objspaces[i]->flags.during_reference_updating = TRUE;
}
for (size_t i = 0; i < global_objspace->global_gc.n_objspaces; i++) {
rb_objspace_t *objspace = global_objspace->global_gc.objspaces[i];
for (int h = 0; h < HEAP_COUNT; h++) {
gc_unprotect_pages(objspace, &heaps[h]);
}
}
rb_gc_before_updating_jit_code();
gc_sweeping_cpu_exit(driver);
for (size_t i = 0; i < global_objspace->global_gc.n_objspaces; i++) {
rb_objspace_t *os = global_objspace->global_gc.objspaces[i];
gc_sweeping_cpu_enter(os);
rb_hrtime_t t0 = (os == driver && driver_prof) ? rb_hrtime_now() : 0;
gc_compact_finish(os);
if (os == driver && driver_prof) {
driver_compact_wall_time = rb_hrtime_add(driver_compact_wall_time, elapsed_hrtime_from(t0));
}
gc_sweeping_cpu_exit(os);
}
/* The VM-global / weak-table side of the reference update runs once (each objspace's
* heap side already ran in gc_compact_finish above). */
{
gc_sweeping_cpu_enter(driver);
rb_hrtime_t t0 = driver_prof ? rb_hrtime_now() : 0;
gc_update_references_global(driver);
if (driver_prof) {
driver_compact_wall_time = rb_hrtime_add(driver_compact_wall_time, elapsed_hrtime_from(t0));
}
gc_sweeping_cpu_exit(driver);
}
gc_sweeping_cpu_enter(driver);
rb_gc_after_updating_jit_code();
for (size_t i = 0; i < global_objspace->global_gc.n_objspaces; i++) {
global_objspace->global_gc.objspaces[i]->flags.during_reference_updating = FALSE;
global_objspace->global_gc.objspaces[i]->flags.during_compacting = FALSE;
}
global_objspace->global_gc.compacting = false;
uninstall_handlers();
gc_sweeping_cpu_exit(driver);
/* Record the driver's compaction time and exclude it from the driver's sweep phase.
* gc_sweeping_exit(driver) in pass 3 subtracts gc_sweep_excluded_wall_time from the
* sweep wall time, so this must be set before it runs. The excluded value is a sum
* of sub-intervals of the driver's sweep phase, so the subtraction cannot underflow. */
if (driver_prof) {
gc_profile_record *const record = gc_prof_record(driver);
record->gc_compact_wall_time = rb_hrtime_add(record->gc_compact_wall_time,
driver_compact_wall_time);
driver->profile.gc_sweep_excluded_wall_time = rb_hrtime_add(
driver->profile.gc_sweep_excluded_wall_time, driver_compact_wall_time);
}
/* pass 3 (free): page-sweep every objspace, freeing dead objects and the source pages
* that are now empty. during_compacting is already cleared, so the sweep treats
* T_MOVED as usual. */
for (size_t i = 0; i < global_objspace->global_gc.n_objspaces; i++) {
rb_objspace_t *os = global_objspace->global_gc.objspaces[i];
gc_sweeping_cpu_enter(os);
gc_sweep_rest(os);
gc_sweeping_exit(os);
}
}
global_objspace->global_gc.compacting = false;
/* A global GC never calls gc_marks_finish, which budgets heap growth
* (allocatable_bytes). An objspace still full after the global sweep (materializing
* a large received copy, say) has no free pages, no empty pages, budget 0, and its next
* allocation would hit newobj_refill's "cannot create a new page after a major GC".
* Give every objspace stuck like that the growth budget gc_marks_finish would. */
for (size_t i = 0; i < global_objspace->global_gc.n_objspaces; i++) {
rb_objspace_t *objspace = global_objspace->global_gc.objspaces[i];
if (objspace->heap_pages.allocatable_bytes != 0 || objspace->empty_pages_count != 0) {
continue;
}
bool stuck = false;
for (int h = 0; h < HEAP_COUNT; h++) {
if (heaps[h].free_pages == NULL) { stuck = true; break; }
}
if (stuck) {
heap_allocatable_bytes_expand(objspace, NULL, 0,
objspace_available_slots(objspace), heaps[0].slot_size);
}
}
/* Recount the surviving shareable objects (the sweep already folded the dead ones out of
* shareable_bits) and reset each trigger limit. */
for (size_t i = 0; i < global_objspace->global_gc.n_objspaces; i++) {
rb_objspace_t *objspace = global_objspace->global_gc.objspaces[i];
size_t survivors = 0;
for (int h = 0; h < HEAP_COUNT; h++) {
struct heap_page *page = NULL;
ccan_list_for_each(&heaps[h].pages, page, page_node) {
if (!page->flags.has_shareable_objects) continue;
for (int j = 0; j < HEAP_PAGE_BITMAP_LIMIT; j++) {
survivors += rb_popcount_intptr(page->shareable_bits[j]);
}
}
}
objspace->shareable_objects = survivors;
size_t new_limit = (size_t)(survivors * SHAREABLE_OBJECTS_LIMIT_FACTOR);
if (new_limit < SHAREABLE_OBJECTS_LIMIT_MIN) new_limit = SHAREABLE_OBJECTS_LIMIT_MIN;
objspace->shareable_objects_limit = new_limit;
}
/* Deferred non-thread-safe frees: the world is already stopped here, so reap them
* without a second barrier. Uses the driver's snapshot rather than taking its own,
* which step 10 below still walks. */
gc_tdata_unsafe_drain_objspaces(global_objspace->global_gc.objspaces,
global_objspace->global_gc.n_objspaces);
/* step 10 */
for (size_t i = 0; i < global_objspace->global_gc.n_objspaces; i++) {
rb_objspace_t *objspace = global_objspace->global_gc.objspaces[i];
objspace->flags.during_global_gc = FALSE;
if (objspace != driver) {
during_gc = FALSE;
gc_process_stat_publish(objspace);
}
}
/* The unified mark re-established the reachability of absorbed shareable objects, so a
* single objspace's local mark is trustworthy again (pinning can be skipped until the
* next absorb). */
rb_gc_reset_absorbed_since_global_gc();
/* Re-measure the zombies now that the garbage is gone. Without this, the page
* trigger above keeps firing on the stale numbers left when a joinable (slotted)
* zombie retires without any pass merging it. */
gc_global_refresh_zombie_pages();
/* If the sweep above collected an unjoined Ractor object, ractor_free disowned its
* zombie_objspaces entry and posted the merge to main as a postponed job; the objspace
* stays enumerable until main absorbs it at its next safepoint. */
gc_prof_timer_stop(driver);
gc_exit(driver, event, &lock_lev);
return true;
}
static int
absorb_finalizer_i(st_data_t key, st_data_t val, st_data_t data)
{
rb_objspace_t *objspace = (rb_objspace_t *)data;
st_insert(finalizer_table, key, val);
return ST_CONTINUE;
}
static void
gc_make_mid_mark_objspace_absorbable(rb_objspace_t *src)
{
rb_objspace_t *objspace = src;
for (int h = 0; h < HEAP_COUNT; h++) {
struct heap_page *page = NULL;
ccan_list_for_each(&heaps[h].pages, page, page_node) {
short stride = page->slot_size;
uintptr_t p = (uintptr_t)page->start;
uintptr_t pend = p + page->total_slots * stride;
for (; p < pend; p += stride) {
VALUE vp = (VALUE)p;
asan_unpoisoning_object(vp) {
switch (RB_BUILTIN_TYPE(vp)) {
case T_NONE:
case T_ZOMBIE:
break;
default:
RVALUE_AGE_RESET(vp);
break;
}
}
}
}
gc_bitmaps_clear(objspace, &heaps[h], false);
}
objspace->rgengc.old_objects = 0;
objspace->rgengc.uncollectible_wb_unprotected_objects = 0;
objspace->marked_slots = 0;
}
/* Merge a dead Ractor's objspace into dst under the VM lock. src has no owner thread and
* dst is the calling thread's own objspace (join/value) or main with everyone stopped
* (global GC), so single-writer holds throughout. Pages move whole (their bits describe
* objects, not the objspace), and dst's next collection is forced full to rebuild the
* generational state. */
static void
objspace_absorb(rb_objspace_t *dst, rb_objspace_t *src)
{
GC_ASSERT(dst != src);
/* Suppress the cross-objspace verifier checks while the graph is in flux (see
* global_objspace->during_absorb). */
const bool prev_absorb = global_objspace->during_absorb;
global_objspace->during_absorb = true;
/* gc.c dropped src from its zombie table in this same VM-lock section, so src went
* through zombify and its pages are part of the total. */
GC_ASSERT(src->zombie_pages != 0);
GC_ASSERT(global_objspace->zombie_total_pages >= src->zombie_pages);
global_objspace->zombie_total_pages -= src->zombie_pages;
src->zombie_pages = 0;
/* Settle dst first: adding pages under a walking lazy-sweep cursor, or into a
* half-marked incremental heap, would sweep the merged pages with src's stale mark
* bits and free live objects. */
gc_rest(dst);
/* Settle src: no incremental marking, lazy sweep or in-progress allocation page. */
{
rb_objspace_t *objspace = src;
/* A zombie parked by fork (rb_ractor_terminate_atfork retires the objspace
* without the retire GC) can be mid-incremental-mark, and that mark cannot be
* finished. Its owner's threads did not survive the fork, so mark_roots would
* scan the absorbing Ractor's roots instead of src's. Drop the cycle. */
if (is_incremental_marking(objspace)) {
gc_abort_incremental_marking(objspace);
gc_make_mid_mark_objspace_absorbable(objspace);
}
/* rb_ractor_terminate_atfork retires without the retire GC, so src may arrive
* with the flag unset; the sweep gc_rest may finish is the last one it gets. */
objspace->flags.ownerless = 1;
gc_rest(src); // if mid-sweep
heap_alloc_state_clear(objspace);
/* gc_sweep_finish leaves swept pages "pooled" for a coming incremental mark; src
* never runs one (it is about to be merged), so return them to its free list now,
* restoring the pooled_pages == NULL the page merge below assumes (mirrors
* gc_start_global step 3). */
for (int h = 0; h < HEAP_COUNT; h++) {
heap_move_pooled_pages_to_free_pages(&heaps[h]);
}
/* Backstop for pages a sweep that ran before `ownerless` was set left attached */
for (int h = 0; h < HEAP_COUNT; h++) {
rb_heap_t *heap = &heaps[h];
struct heap_page **link = &heap->free_pages;
while (*link) {
struct heap_page *page = *link;
if (page->free_slots == page->total_slots) {
*link = page->free_next;
}
else {
link = &page->free_next;
}
}
struct heap_page *page, *next;
ccan_list_for_each_safe(&heap->pages, page, next, page_node) {
if (page->free_slots == page->total_slots) {
GC_ASSERT(page->final_slots == 0);
heap_page_detach_empty(objspace, heap, page);
}
}
}
}
/* From here the merge must not run dst's GC: the finalizer st_insert below can cross
* the malloc-accounting threshold, and a GC then would sweep src's detached finalizer
* procs, reachable only from this C frame, into dangling VALUEs. Page/darray moves
* allocate nothing (_without_gc), so disabling costs nothing and makes the splice
* atomic. (The two settles above deliberately collect: they stay outside.) */
const bool dst_gc_was_enabled = rb_gc_impl_gc_enabled_p(dst);
if (dst_gc_was_enabled) rb_gc_impl_gc_disable(dst, false);
/* Hand over the pages size pool by size pool. ("heaps" is a macro over the local
* objspace, so the arrays are taken through scoped locals.) */
rb_heap_t *dst_heaps;
rb_heap_t *src_heaps;
{
rb_objspace_t *objspace = dst;
dst_heaps = heaps;
}
{
rb_objspace_t *objspace = src;
src_heaps = heaps;
}
for (int h = 0; h < HEAP_COUNT; h++) {
rb_heap_t *dheap = &dst_heaps[h];
rb_heap_t *sheap = &src_heaps[h];
struct heap_page *page = NULL;
GC_ASSERT(sheap->sweeping_page == NULL);
GC_ASSERT(sheap->pooled_pages == NULL);
ccan_list_for_each(&sheap->pages, page, page_node) {
page->objspace = dst;
page->heap = dheap;
}
ccan_list_append_list(&dheap->pages, &sheap->pages);
/* Append the free-page chain to the tail. */
if (sheap->free_pages) {
struct heap_page **tail = &dheap->free_pages;
while (*tail) tail = &(*tail)->free_next;
*tail = sheap->free_pages;
sheap->free_pages = NULL;
}
dheap->total_pages += sheap->total_pages;
dheap->total_slots += sheap->total_slots;
dheap->total_allocated_pages += sheap->total_allocated_pages;
dheap->total_allocated_objects += sheap->total_allocated_objects;
dheap->total_freed_objects += sheap->total_freed_objects;
dheap->final_slots_count += sheap->final_slots_count;
}
/* The objspace-wide page bookkeeping. */
{
rb_objspace_t *objspace = dst; /* for the heap_pages_* macros */
struct heap_page *page = NULL;
size_t srcn = rb_darray_size(src->heap_pages.sorted);
for (size_t i = 0; i < srcn; i++) {
page = rb_darray_get(src->heap_pages.sorted, i);
/* Residents of the empty pool (no live objects) are returned to page_pool rather
* than inherited; dst's allocation demand is cheaply met from the shared pool's
* free list. */
if (heap_page_in_global_empty_pages_pool(src, page)) {
heap_page_free(src, page);
continue;
}
uintptr_t body = (uintptr_t)page->body;
uintptr_t start = body + sizeof(struct heap_page_header);
uintptr_t end = body + HEAP_PAGE_SIZE;
/* Keep the array ordered by page BODY address: heap_page_for_ptr bsearches
* body ranges, and a detached empty page has start == 0, so ordering by
* page->start would miss live pages (a global GC would then fail to mark a
* registered root and sweep it). */
size_t lo = 0;
size_t hi = rb_darray_size(objspace->heap_pages.sorted);
while (lo < hi) {
size_t mid = (lo + hi) / 2;
struct heap_page *mid_page = rb_darray_get(objspace->heap_pages.sorted, mid);
if ((uintptr_t)mid_page->body < body) lo = mid + 1;
else hi = mid;
}
rb_darray_insert_without_gc(&objspace->heap_pages.sorted, hi, page);
if (heap_pages_lomem == 0 || heap_pages_lomem > start) heap_pages_lomem = start;
if (heap_pages_himem < end) heap_pages_himem = end;
}
objspace->heap_pages.allocated_pages += src->heap_pages.allocated_pages;
objspace->heap_pages.freed_pages += src->heap_pages.freed_pages;
rb_darray_free_without_gc(src->heap_pages.sorted);
src->heap_pages.sorted = NULL;
/* The empty_pages chain's structs were freed in the loop above. */
src->empty_pages = NULL;
src->empty_pages_count = 0;
}
/* Finalizers: move the table's entries, and the dead Ractor's deferred zombies are run
* by dst's thread from now on. */
{
st_table *src_finalizers;
{
rb_objspace_t *objspace = src;
src_finalizers = finalizer_table;
finalizer_table = NULL;
}
if (src_finalizers) {
rb_objspace_t *objspace = dst;
if (finalizer_table == NULL) {
finalizer_table = src_finalizers;
}
else {
st_foreach(src_finalizers, absorb_finalizer_i, (st_data_t)dst);
st_free_table(src_finalizers);
}
}
}
{
VALUE src_deferred = RUBY_ATOMIC_VALUE_EXCHANGE(src->heap_pages.deferred_final, 0);
if (src_deferred) {
VALUE tail_obj = src_deferred;
rb_asan_unpoison_object(tail_obj, false);
while (RZOMBIE(tail_obj)->next) {
VALUE next_obj = RZOMBIE(tail_obj)->next;
rb_asan_poison_object(tail_obj);
tail_obj = next_obj;
rb_asan_unpoison_object(tail_obj, false);
}
VALUE prev;
do {
prev = dst->heap_pages.deferred_final;
RZOMBIE(tail_obj)->next = prev;
} while (RUBY_ATOMIC_VALUE_CAS(dst->heap_pages.deferred_final, prev, src_deferred) != prev);
rb_asan_poison_object(tail_obj);
/* No owner was left to run these zombies (register's owner walk misses a dead
* Ractor). dst runs this merge, so schedule dst's job here; otherwise they wait
* until dst's next GC. */
rb_postponed_job_trigger(dst->finalize_deferred_pjob);
}
}
if (src->tdata_unsafe_free_chunk) {
gc_tdata_unsafe_free_publish(src);
}
if (tdata_deferred_free_count_load() >= TDATA_DEFERRED_FREE_THRESHOLD) {
gc_tdata_deferred_free_trigger(dst);
}
/* Counters inherited by dst. */
dst->rgengc.old_objects += src->rgengc.old_objects;
dst->rgengc.uncollectible_wb_unprotected_objects += src->rgengc.uncollectible_wb_unprotected_objects;
dst->shareable_objects += src->shareable_objects;
/* Merged pages carry src's mark/age state, so dst rebuilds its view at the next
* collection. Note that this can be worked around by calling `GC.config(rgengc_allow_full_mark: false)`,
* so the absorbed heap should be in a state where a minor GC would also work correctly.
*/
dst->rgengc.need_major_gc |= GPR_FLAG_MAJOR_BY_FORCE;
/* src's outstanding malloc pressure moves with the xmalloc'd buffers. Later frees are
* charged to dst, so without this transfer dst underestimates its own heap and delays
* GCs. dst is live, so take its counter lock where gc_counter_add is not atomic. */
{
int64_t inc = gc_malloc_counters_increase(src, &src->malloc_counters.counters);
#if RGENGC_ESTIMATE_OLDMALLOC
size_t oldmalloc_increase = gc_oldmalloc_increase_load(src);
#endif
MALLOC_COUNTERS_LOCK(dst);
if (inc > 0) gc_counter_add(&dst->malloc_counters.counters.malloc, (size_t)inc);
#if RGENGC_ESTIMATE_OLDMALLOC
if (oldmalloc_increase > 0) RUBY_ATOMIC_SIZE_ADD(dst->malloc_counters.oldmalloc_increase, oldmalloc_increase);
#endif
MALLOC_COUNTERS_UNLOCK(dst);
}
{
struct gc_process_stat_snapshot final_snap;
gc_process_stat_capture(src, &final_snap);
gc_process_stat_add(&global_objspace->process_stat_archive, &final_snap);
}
rb_native_mutex_destroy(&src->process_stat.lock);
/* Free the shell (as rb_gc_impl_objspace_free does). */
free(src->profile.records);
free_stack_chunks(&src->mark_stack);
mark_stack_free_cache(&src->mark_stack);
GC_ASSERT(rb_darray_size(src->weak_references) == 0);
rb_darray_free_without_gc(src->weak_references);
#ifdef MALLOC_COUNTERS_NEED_LOCK
rb_native_mutex_destroy(&src->malloc_counters.lock);
#endif
free(src);
if (dst_gc_was_enabled) rb_gc_impl_gc_enable(dst);
/* Return the empty pages inheritance piled up in dst (mostly from the dead Ractor's
* teardown material) to the pool with no budget. An empty page is by definition safe to
* release, and re-acquiring one from the pool is cheap. */
{
rb_objspace_t *objspace = dst;
heap_pages_freeable_pages = objspace->empty_pages_count;
heap_pages_free_unused_pages(objspace);
}
global_objspace->during_absorb = prev_absorb;
}
void
rb_gc_impl_objspace_absorb(void *dst_ptr, void *src_ptr)
{
objspace_absorb(dst_ptr, src_ptr);
}
void
rb_gc_impl_start(void *objspace_ptr, bool full_mark, bool immediate_mark, bool immediate_sweep, bool compact, bool global)
{
rb_objspace_t *objspace = objspace_ptr;
unsigned int reason = (GPR_FLAG_FULL_MARK |
GPR_FLAG_IMMEDIATE_MARK |
GPR_FLAG_IMMEDIATE_SWEEP |
GPR_FLAG_METHOD);
int full_marking_p = gc_config_full_mark_val;
gc_config_full_mark_set(TRUE);
/* For now, compact implies full mark / sweep, so ignore other flags */
if (compact) {
GC_ASSERT(GC_COMPACTION_SUPPORTED);
reason |= GPR_FLAG_COMPACT;
if (!rb_gc_single_objspace_p()) {
global = true;
}
}
else {
if (!full_mark) reason &= ~GPR_FLAG_FULL_MARK;
if (!immediate_mark) reason &= ~GPR_FLAG_IMMEDIATE_MARK;
if (!immediate_sweep) reason &= ~GPR_FLAG_IMMEDIATE_SWEEP;
}
if ((reason & (GPR_FLAG_FULL_MARK | GPR_FLAG_IMMEDIATE_MARK | GPR_FLAG_IMMEDIATE_SWEEP)) !=
(GPR_FLAG_FULL_MARK | GPR_FLAG_IMMEDIATE_MARK | GPR_FLAG_IMMEDIATE_SWEEP)) {
global = false;
}
if (global && !rb_gc_single_objspace_p()) {
/* A mid-cycle driver is settled by gc_start_global itself: it aborts the partial
* mark and finishes the lazy sweep, so the dead slots are T_NONE before the
* unified conservative root scan. */
gc_start_global(objspace, reason, compact || ruby_enable_autocompact, false);
}
else {
garbage_collect(objspace, reason);
}
gc_finalize_deferred(objspace);
/* An explicit GC.start is expected to reclaim immediately, so run the deferred non-thread-safe
* frees synchronously instead of leaving them to gc_sweep_finish's postponed job. */
if (tdata_deferred_free_count_load() > 0 && rb_gc_single_objspace_p()) {
gc_tdata_unsafe_drain();
}
gc_config_full_mark_set(full_marking_p);
}
void
rb_gc_impl_prepare_heap(void *objspace_ptr)
{
rb_objspace_t *objspace = objspace_ptr;
rb_gc_impl_each_objects(objspace, gc_set_candidate_object_i, objspace_ptr);
double orig_max_free_slots = gc_params.heap_free_slots_max_ratio;
/* Ensure that all empty pages are moved onto empty_pages. */
gc_params.heap_free_slots_max_ratio = 0.0;
rb_gc_impl_start(objspace, true, true, true, true, true);
gc_params.heap_free_slots_max_ratio = orig_max_free_slots;
objspace->heap_pages.allocatable_bytes = 0;
heap_pages_freeable_pages = objspace->empty_pages_count;
heap_pages_free_unused_pages(objspace_ptr);
GC_ASSERT(heap_pages_freeable_pages == 0);
GC_ASSERT(objspace->empty_pages_count == 0);
// Process.warmup is meant to be called at the end of the boot sequence, which is commonly allocation
// heavy and result in GC limits raising significantly, but it's not indicative of the limits needed
// for runtime.
// Recompute the allocatable_bytes limit based on `gc_params.heap_init_bytes`.
GC_ASSERT(objspace->heap_pages.allocatable_bytes == 0);
for (int i = 0; i < HEAP_COUNT; i++) {
rb_heap_t *heap = &heaps[i];
heap_allocatable_bytes_expand(objspace, heap, heap->empty_slots, heap->total_slots, heap->slot_size);
}
#if defined(HAVE_MALLOC_TRIM) && !defined(RUBY_ALTERNATIVE_MALLOC_HEADER)
malloc_trim(0);
#endif
}
static int
gc_is_moveable_obj(rb_objspace_t *objspace, VALUE obj)
{
GC_ASSERT(!SPECIAL_CONST_P(obj));
switch (BUILTIN_TYPE(obj)) {
case T_NONE:
case T_MOVED:
case T_ZOMBIE:
return FALSE;
case T_SYMBOL:
case T_STRING:
case T_OBJECT:
case T_FLOAT:
case T_IMEMO:
case T_ARRAY:
case T_BIGNUM:
case T_ICLASS:
case T_MODULE:
case T_REGEXP:
case T_DATA:
case T_MATCH:
case T_STRUCT:
case T_HASH:
case T_FILE:
case T_COMPLEX:
case T_RATIONAL:
case T_NODE:
case T_CLASS:
if (FL_TEST_RAW(obj, FL_FINALIZE)) {
/* The finalizer table is a numtable. It looks up objects by address.
* We can't mark the keys in the finalizer table because that would
* prevent the objects from being collected. This check prevents
* objects that are keys in the finalizer table from being moved
* without directly pinning them. */
GC_ASSERT(st_is_member(finalizer_table, obj));
return FALSE;
}
GC_ASSERT(RVALUE_MARKED(objspace, obj));
GC_ASSERT(!RVALUE_PINNED(objspace, obj));
return TRUE;
default:
rb_bug("gc_is_moveable_obj: unreachable (%d)", (int)BUILTIN_TYPE(obj));
break;
}
return FALSE;
}
void rb_mv_generic_ivar(VALUE src, VALUE dst);
static VALUE
gc_move(rb_objspace_t *objspace, VALUE src, VALUE dest, struct heap_page *src_page, struct heap_page *dest_page)
{
size_t src_slot_size = src_page->slot_size;
size_t slot_size = dest_page->slot_size;
int marked;
int wb_unprotected;
int uncollectible;
int age;
gc_report(4, objspace, "Moving object: %p -> %p\n", (void *)src, (void *)dest);
GC_ASSERT(BUILTIN_TYPE(src) != T_NONE);
GC_ASSERT(!MARKED_IN_BITMAP(GET_HEAP_MARK_BITS(dest), dest));
GC_ASSERT(!RVALUE_MARKING(objspace, src));
/* Save off bits for current object. */
marked = RVALUE_MARKED(objspace, src);
wb_unprotected = RVALUE_WB_UNPROTECTED(objspace, src);
uncollectible = RVALUE_UNCOLLECTIBLE(objspace, src);
bool remembered = RVALUE_REMEMBERED(objspace, src);
/* Pin bits travel with the object. Losing one during single-objspace compaction would
* silently unpin it once the process goes multi-objspace, letting a local GC free a method
* entry or shref target that another Ractor references. */
bool shareable = MARKED_IN_BITMAP(GET_HEAP_SHAREABLE_BITS(src), src) != 0;
bool shref = MARKED_IN_BITMAP(GET_HEAP_SHREF_BITS(src), src) != 0;
age = RVALUE_AGE_GET(src);
/* Clear bits for eventual T_MOVED */
CLEAR_IN_BITMAP(GET_HEAP_MARK_BITS(src), src);
CLEAR_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(src), src);
CLEAR_IN_BITMAP(GET_HEAP_UNCOLLECTIBLE_BITS(src), src);
CLEAR_IN_BITMAP(GET_HEAP_PAGE(src)->remembered_bits, src);
CLEAR_IN_BITMAP(GET_HEAP_SHAREABLE_BITS(src), src);
CLEAR_IN_BITMAP(GET_HEAP_SHREF_BITS(src), src);
/* Move the object */
memcpy((void *)dest, (void *)src, MIN(src_slot_size, slot_size));
if (src_slot_size != slot_size) {
rb_gc_obj_changed_slot_size(dest, slot_size - RVALUE_OVERHEAD);
}
if (RVALUE_OVERHEAD > 0) {
void *dest_overhead = (void *)(((uintptr_t)dest) + slot_size - RVALUE_OVERHEAD);
void *src_overhead = (void *)(((uintptr_t)src) + src_slot_size - RVALUE_OVERHEAD);
memcpy(dest_overhead, src_overhead, RVALUE_OVERHEAD);
}
memset((void *)src, 0, src_slot_size);
RVALUE_AGE_SET_BITMAP(src, 0);
/* Set bits for object in new location */
if (remembered) {
MARK_IN_BITMAP(GET_HEAP_PAGE(dest)->remembered_bits, dest);
}
else {
CLEAR_IN_BITMAP(GET_HEAP_PAGE(dest)->remembered_bits, dest);
}
if (marked) {
MARK_IN_BITMAP(GET_HEAP_MARK_BITS(dest), dest);
}
else {
CLEAR_IN_BITMAP(GET_HEAP_MARK_BITS(dest), dest);
}
if (wb_unprotected) {
MARK_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(dest), dest);
}
else {
CLEAR_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(dest), dest);
}
if (uncollectible) {
MARK_IN_BITMAP(GET_HEAP_UNCOLLECTIBLE_BITS(dest), dest);
}
else {
CLEAR_IN_BITMAP(GET_HEAP_UNCOLLECTIBLE_BITS(dest), dest);
}
if (shareable) {
MARK_IN_BITMAP(GET_HEAP_SHAREABLE_BITS(dest), dest);
GET_HEAP_PAGE(dest)->flags.has_shareable_objects = TRUE;
}
else {
CLEAR_IN_BITMAP(GET_HEAP_SHAREABLE_BITS(dest), dest);
}
if (shref) {
MARK_IN_BITMAP(GET_HEAP_SHREF_BITS(dest), dest);
GET_HEAP_PAGE(dest)->flags.has_shref_objects = TRUE;
}
else {
CLEAR_IN_BITMAP(GET_HEAP_SHREF_BITS(dest), dest);
}
RVALUE_AGE_SET(dest, age);
/* A re-embedded object (rb_gc_obj_changed_slot_size) references its
* former fields_obj's contents directly; the write-barrier history
* lived on the discarded fields_obj, so remember the object. */
if (src_slot_size != slot_size && age >= RVALUE_OLD_AGE && !remembered) {
rgengc_remember(objspace, dest);
}
/* Assign forwarding address */
RMOVED(src)->flags = T_MOVED;
RMOVED(src)->dummy = Qundef;
RMOVED(src)->destination = dest;
GC_ASSERT(BUILTIN_TYPE(dest) != T_NONE);
GET_HEAP_PAGE(src)->heap->total_freed_objects++;
GET_HEAP_PAGE(dest)->heap->total_allocated_objects++;
return src;
}
#if GC_CAN_COMPILE_COMPACTION
static int
compare_pinned_slots(const void *left, const void *right, void *dummy)
{
struct heap_page *left_page;
struct heap_page *right_page;
left_page = *(struct heap_page * const *)left;
right_page = *(struct heap_page * const *)right;
return left_page->pinned_slots - right_page->pinned_slots;
}
static int
compare_free_slots(const void *left, const void *right, void *dummy)
{
struct heap_page *left_page;
struct heap_page *right_page;
left_page = *(struct heap_page * const *)left;
right_page = *(struct heap_page * const *)right;
return left_page->free_slots - right_page->free_slots;
}
static void
gc_sort_heap_by_compare_func(rb_objspace_t *objspace, gc_compact_compare_func compare_func)
{
for (int j = 0; j < HEAP_COUNT; j++) {
rb_heap_t *heap = &heaps[j];
size_t total_pages = heap->total_pages;
size_t size = rb_size_mul_or_raise(total_pages, sizeof(struct heap_page *), rb_eRuntimeError);
struct heap_page *page = 0, **page_list = malloc(size);
size_t i = 0;
heap->free_pages = NULL;
ccan_list_for_each(&heap->pages, page, page_node) {
page_list[i++] = page;
GC_ASSERT(page);
}
GC_ASSERT((size_t)i == total_pages);
/* Sort the heap so "filled pages" are first. `heap_add_page` adds to the
* head of the list, so empty pages will end up at the start of the heap */
ruby_qsort(page_list, total_pages, sizeof(struct heap_page *), compare_func, NULL);
/* Reset the eden heap */
ccan_list_head_init(&heap->pages);
for (i = 0; i < total_pages; i++) {
ccan_list_add(&heap->pages, &page_list[i]->page_node);
if (page_list[i]->free_slots != 0) {
heap_add_freepage(heap, page_list[i]);
}
}
free(page_list);
}
}
#endif
void
rb_gc_impl_register_pinning_obj(void *objspace_ptr, VALUE obj)
{
/* no-op */
}
bool
rb_gc_impl_object_moved_p(void *objspace_ptr, VALUE obj)
{
return gc_object_moved_p(objspace_ptr, obj);
}
static int
gc_ref_update(void *vstart, void *vend, size_t stride, rb_objspace_t *objspace, struct heap_page *page)
{
VALUE v = (VALUE)vstart;
page->flags.has_uncollectible_wb_unprotected_objects = FALSE;
page->flags.has_remembered_objects = FALSE;
/* For each object on the page */
for (; v != (VALUE)vend; v += stride) {
asan_unpoisoning_object(v) {
switch (BUILTIN_TYPE(v)) {
case T_NONE:
case T_MOVED:
case T_ZOMBIE:
break;
default:
if (RVALUE_WB_UNPROTECTED(objspace, v)) {
page->flags.has_uncollectible_wb_unprotected_objects = TRUE;
}
if (RVALUE_REMEMBERED(objspace, v)) {
page->flags.has_remembered_objects = TRUE;
}
if (page->flags.before_sweep) {
if (RVALUE_MARKED(objspace, v)) {
rb_gc_update_object_references(objspace, v);
}
}
else {
rb_gc_update_object_references(objspace, v);
}
}
}
}
return 0;
}
static int
gc_update_references_weak_table_i(VALUE obj, void *data)
{
int ret;
asan_unpoisoning_object(obj) {
ret = BUILTIN_TYPE(obj) == T_MOVED ? ST_REPLACE : ST_CONTINUE;
}
return ret;
}
static int
gc_update_references_weak_table_replace_i(VALUE *obj, void *data)
{
rb_gc_update_moved(obj);
return ST_CONTINUE;
}
/* The per-objspace side of the reference update: walk this objspace's heap objects and rewrite
* moved references (following T_MOVED forwarding across objspaces). A compacting global GC
* runs this for every objspace. */
static void
gc_update_references_heap(rb_objspace_t *objspace)
{
struct heap_page *page = NULL;
for (int i = 0; i < HEAP_COUNT; i++) {
bool should_set_mark_bits = TRUE;
rb_heap_t *heap = &heaps[i];
ccan_list_for_each(&heap->pages, page, page_node) {
uintptr_t start = (uintptr_t)page->start;
uintptr_t end = start + (page->total_slots * heap->slot_size);
gc_ref_update((void *)start, (void *)end, heap->slot_size, objspace, page);
if (page == heap->sweeping_page) {
should_set_mark_bits = FALSE;
}
if (should_set_mark_bits) {
gc_setup_mark_bits(page);
}
}
}
}
/* The VM-global side of the reference update (finalizer table, every Ractor's VM roots,
* weak tables). Process-wide, so a compacting global GC runs it once after every heap
* side: rb_gc_update_vm_references and the weak tables' mark_and_move are not idempotent. */
static void
gc_update_references_global(rb_objspace_t *objspace)
{
gc_update_table_refs(finalizer_table);
rb_gc_update_vm_references((void *)objspace);
for (int table = 0; table < RB_GC_VM_WEAK_TABLE_COUNT; table++) {
rb_gc_vm_weak_table_foreach(
gc_update_references_weak_table_i,
gc_update_references_weak_table_replace_i,
NULL,
false,
table
);
}
}
static void
gc_update_references(rb_objspace_t *objspace)
{
objspace->flags.during_reference_updating = true;
rb_gc_before_updating_jit_code();
gc_update_references_heap(objspace);
gc_update_references_global(objspace);
rb_gc_after_updating_jit_code();
objspace->flags.during_reference_updating = false;
}
#if GC_CAN_COMPILE_COMPACTION
static void
root_obj_check_moved_i(const char *category, VALUE obj, void *data)
{
rb_objspace_t *objspace = data;
if (gc_object_moved_p(objspace, obj)) {
rb_bug("ROOT %s points to MOVED: %p -> %s", category, (void *)obj, rb_obj_info(rb_gc_impl_location(objspace, obj)));
}
}
static void
reachable_object_check_moved_i(VALUE ref, void *data)
{
VALUE parent = (VALUE)data;
if (gc_object_moved_p(rb_gc_get_objspace(), ref)) {
rb_bug("Object %s points to MOVED: %p -> %s", rb_obj_info(parent), (void *)ref, rb_obj_info(rb_gc_impl_location(rb_gc_get_objspace(), ref)));
}
}
static int
heap_check_moved_i(void *vstart, void *vend, size_t stride, void *data)
{
rb_objspace_t *objspace = data;
VALUE v = (VALUE)vstart;
for (; v != (VALUE)vend; v += stride) {
if (gc_object_moved_p(objspace, v)) {
/* Moved object still on the heap, something may have a reference. */
}
else {
asan_unpoisoning_object(v) {
switch (BUILTIN_TYPE(v)) {
case T_NONE:
case T_ZOMBIE:
break;
default:
if (!rb_gc_impl_garbage_object_p(objspace, v)) {
rb_objspace_reachable_objects_from_unlocked(v, reachable_object_check_moved_i, (void *)v);
}
}
}
}
}
return 0;
}
#endif
bool
rb_gc_impl_during_gc_p(void *objspace_ptr)
{
rb_objspace_t *objspace = objspace_ptr;
return during_gc;
}
#if RGENGC_PROFILE >= 2
static const char*
type_name(int type, VALUE obj)
{
switch ((enum ruby_value_type)type) {
case RUBY_T_NONE: return "T_NONE";
case RUBY_T_OBJECT: return "T_OBJECT";
case RUBY_T_CLASS: return "T_CLASS";
case RUBY_T_MODULE: return "T_MODULE";
case RUBY_T_FLOAT: return "T_FLOAT";
case RUBY_T_STRING: return "T_STRING";
case RUBY_T_REGEXP: return "T_REGEXP";
case RUBY_T_ARRAY: return "T_ARRAY";
case RUBY_T_HASH: return "T_HASH";
case RUBY_T_STRUCT: return "T_STRUCT";
case RUBY_T_BIGNUM: return "T_BIGNUM";
case RUBY_T_FILE: return "T_FILE";
case RUBY_T_DATA: return "T_DATA";
case RUBY_T_MATCH: return "T_MATCH";
case RUBY_T_COMPLEX: return "T_COMPLEX";
case RUBY_T_RATIONAL: return "T_RATIONAL";
case RUBY_T_NIL: return "T_NIL";
case RUBY_T_TRUE: return "T_TRUE";
case RUBY_T_FALSE: return "T_FALSE";
case RUBY_T_SYMBOL: return "T_SYMBOL";
case RUBY_T_FIXNUM: return "T_FIXNUM";
case RUBY_T_UNDEF: return "T_UNDEF";
case RUBY_T_IMEMO: return "T_IMEMO";
case RUBY_T_NODE: return "T_NODE";
case RUBY_T_ICLASS: return "T_ICLASS";
case RUBY_T_ZOMBIE: return "T_ZOMBIE";
case RUBY_T_MOVED: return "T_MOVED";
default: return "unknown";
}
}
static void
gc_count_add_each_types(VALUE hash, const char *name, const size_t *types)
{
VALUE result = rb_hash_new_capa(T_MASK);
int i;
for (i=0; i<T_MASK; i++) {
const char *type = type_name(i, 0);
rb_hash_aset(result, ID2SYM(rb_intern(type)), SIZET2NUM(types[i]));
}
rb_hash_aset(hash, ID2SYM(rb_intern(name)), result);
}
#endif
size_t
rb_gc_impl_gc_count(void *objspace_ptr)
{
rb_objspace_t *objspace = objspace_ptr;
return objspace->profile.count;
}
/* Filled by setup_gc_latest_gc_info_symbols() at boot, not on first use. */
static VALUE sym_major_by, sym_gc_by, sym_immediate_sweep, sym_have_finalizer, sym_state, sym_need_major_by;
static VALUE sym_nofree, sym_oldgen, sym_shady, sym_force, sym_stress;
#if RGENGC_ESTIMATE_OLDMALLOC
static VALUE sym_oldmalloc;
#endif
static VALUE sym_newobj, sym_malloc, sym_method, sym_capi;
static VALUE sym_none, sym_marking, sym_sweeping;
static VALUE sym_weak_references_count;
static void
setup_gc_latest_gc_info_symbols(void)
{
#define S(s) sym_##s = ID2SYM(rb_intern_const(#s))
S(major_by);
S(gc_by);
S(immediate_sweep);
S(have_finalizer);
S(state);
S(need_major_by);
S(stress);
S(nofree);
S(oldgen);
S(shady);
S(force);
#if RGENGC_ESTIMATE_OLDMALLOC
S(oldmalloc);
#endif
S(newobj);
S(malloc);
S(method);
S(capi);
S(none);
S(marking);
S(sweeping);
S(weak_references_count);
#undef S
}
static VALUE
gc_info_decode(rb_objspace_t *objspace, const VALUE hash_or_key, const unsigned int orig_flags)
{
VALUE hash = Qnil, key = Qnil;
VALUE major_by, need_major_by;
unsigned int flags = orig_flags ? orig_flags : objspace->profile.latest_gc_info;
if (SYMBOL_P(hash_or_key)) {
key = hash_or_key;
}
else if (RB_TYPE_P(hash_or_key, T_HASH)) {
hash = hash_or_key;
}
else {
rb_bug("gc_info_decode: non-hash or symbol given");
}
#define SET(name, attr) \
if (key == sym_##name) \
return (attr); \
else if (hash != Qnil) \
rb_hash_aset(hash, sym_##name, (attr));
major_by =
(flags & GPR_FLAG_MAJOR_BY_NOFREE) ? sym_nofree :
(flags & GPR_FLAG_MAJOR_BY_OLDGEN) ? sym_oldgen :
(flags & GPR_FLAG_MAJOR_BY_SHADY) ? sym_shady :
(flags & GPR_FLAG_MAJOR_BY_FORCE) ? sym_force :
#if RGENGC_ESTIMATE_OLDMALLOC
(flags & GPR_FLAG_MAJOR_BY_OLDMALLOC) ? sym_oldmalloc :
#endif
Qnil;
SET(major_by, major_by);
if (orig_flags == 0) { /* set need_major_by only if flags not set explicitly */
unsigned int need_major_flags = gc_needs_major_flags;
need_major_by =
(need_major_flags & GPR_FLAG_MAJOR_BY_NOFREE) ? sym_nofree :
(need_major_flags & GPR_FLAG_MAJOR_BY_OLDGEN) ? sym_oldgen :
(need_major_flags & GPR_FLAG_MAJOR_BY_SHADY) ? sym_shady :
(need_major_flags & GPR_FLAG_MAJOR_BY_FORCE) ? sym_force :
#if RGENGC_ESTIMATE_OLDMALLOC
(need_major_flags & GPR_FLAG_MAJOR_BY_OLDMALLOC) ? sym_oldmalloc :
#endif
Qnil;
SET(need_major_by, need_major_by);
}
SET(gc_by,
(flags & GPR_FLAG_NEWOBJ) ? sym_newobj :
(flags & GPR_FLAG_MALLOC) ? sym_malloc :
(flags & GPR_FLAG_METHOD) ? sym_method :
(flags & GPR_FLAG_CAPI) ? sym_capi :
(flags & GPR_FLAG_STRESS) ? sym_stress :
Qnil
);
SET(have_finalizer, (flags & GPR_FLAG_HAVE_FINALIZE) ? Qtrue : Qfalse);
SET(immediate_sweep, (flags & GPR_FLAG_IMMEDIATE_SWEEP) ? Qtrue : Qfalse);
if (orig_flags == 0) {
SET(state, gc_mode(objspace) == gc_mode_none ? sym_none :
gc_mode(objspace) == gc_mode_marking ? sym_marking : sym_sweeping);
}
SET(weak_references_count, LONG2FIX(objspace->profile.weak_references_count));
#undef SET
if (!NIL_P(key)) {
// Matched key should return above
return Qundef;
}
return hash;
}
VALUE
rb_gc_impl_latest_gc_info(void *objspace_ptr, VALUE key)
{
rb_objspace_t *objspace = objspace_ptr;
return gc_info_decode(objspace, key, 0);
}
enum gc_stat_sym {
gc_stat_sym_count,
gc_stat_sym_time,
gc_stat_sym_marking_time,
gc_stat_sym_sweeping_time,
gc_stat_sym_heap_allocated_pages,
gc_stat_sym_heap_empty_pages,
gc_stat_sym_heap_allocatable_bytes,
gc_stat_sym_heap_available_slots,
gc_stat_sym_heap_live_slots,
gc_stat_sym_heap_free_slots,
gc_stat_sym_heap_final_slots,
gc_stat_sym_heap_marked_slots,
gc_stat_sym_heap_eden_pages,
gc_stat_sym_total_allocated_pages,
gc_stat_sym_total_freed_pages,
gc_stat_sym_total_allocated_objects,
gc_stat_sym_total_freed_objects,
gc_stat_sym_total_malloc_bytes,
gc_stat_sym_total_free_bytes,
gc_stat_sym_malloc_increase_bytes,
gc_stat_sym_malloc_increase_bytes_limit,
gc_stat_sym_minor_gc_count,
gc_stat_sym_major_gc_count,
gc_stat_sym_global_gc_count,
gc_stat_sym_global_gc_invocation_count,
gc_stat_sym_compact_count,
gc_stat_sym_read_barrier_faults,
gc_stat_sym_total_moved_objects,
gc_stat_sym_remembered_wb_unprotected_objects,
gc_stat_sym_remembered_wb_unprotected_objects_limit,
gc_stat_sym_old_objects,
gc_stat_sym_old_objects_limit,
#if RGENGC_ESTIMATE_OLDMALLOC
gc_stat_sym_oldmalloc_increase_bytes,
gc_stat_sym_oldmalloc_increase_bytes_limit,
#endif
#if RGENGC_PROFILE
gc_stat_sym_total_generated_normal_object_count,
gc_stat_sym_total_generated_shady_object_count,
gc_stat_sym_total_shade_operation_count,
gc_stat_sym_total_promoted_count,
gc_stat_sym_total_remembered_normal_object_count,
gc_stat_sym_total_remembered_shady_object_count,
#endif
gc_stat_sym_page_pool_arenas,
gc_stat_sym_page_pool_arenas_freed,
gc_stat_sym_page_pool_total_pages,
gc_stat_sym_page_pool_discarded_pages,
gc_stat_sym_last
};
static VALUE gc_stat_symbols[gc_stat_sym_last];
static void
setup_gc_stat_symbols(void)
{
#define S(s) gc_stat_symbols[gc_stat_sym_##s] = ID2SYM(rb_intern_const(#s))
S(count);
S(time);
S(marking_time),
S(sweeping_time),
S(heap_allocated_pages);
S(heap_empty_pages);
S(heap_allocatable_bytes);
S(heap_available_slots);
S(heap_live_slots);
S(heap_free_slots);
S(heap_final_slots);
S(heap_marked_slots);
S(heap_eden_pages);
S(total_allocated_pages);
S(total_freed_pages);
S(total_allocated_objects);
S(total_freed_objects);
S(total_malloc_bytes);
S(total_free_bytes);
S(malloc_increase_bytes);
S(malloc_increase_bytes_limit);
S(minor_gc_count);
S(major_gc_count);
S(global_gc_count);
S(global_gc_invocation_count);
S(compact_count);
S(read_barrier_faults);
S(total_moved_objects);
S(remembered_wb_unprotected_objects);
S(remembered_wb_unprotected_objects_limit);
S(old_objects);
S(old_objects_limit);
#if RGENGC_ESTIMATE_OLDMALLOC
S(oldmalloc_increase_bytes);
S(oldmalloc_increase_bytes_limit);
#endif
#if RGENGC_PROFILE
S(total_generated_normal_object_count);
S(total_generated_shady_object_count);
S(total_shade_operation_count);
S(total_promoted_count);
S(total_remembered_normal_object_count);
S(total_remembered_shady_object_count);
#endif /* RGENGC_PROFILE */
S(page_pool_arenas);
S(page_pool_arenas_freed);
S(page_pool_total_pages);
S(page_pool_discarded_pages);
#undef S
}
static uint64_t
ns_to_ms(uint64_t ns)
{
return ns / (1000 * 1000);
}
static void malloc_increase_local_flush(rb_objspace_t *objspace);
static void
gc_process_stat_accumulate_i(void *objspace_ptr, void *data)
{
rb_objspace_t *objspace = objspace_ptr;
struct gc_process_stat_total *total = (struct gc_process_stat_total *)data;
struct gc_process_stat_snapshot snap;
rb_native_mutex_lock(&objspace->process_stat.lock);
snap = objspace->process_stat.published;
rb_native_mutex_unlock(&objspace->process_stat.lock);
gc_process_stat_add(total, &snap);
}
static VALUE
gc_process_stat(VALUE hash_or_sym)
{
VALUE hash = Qnil, key = Qnil;
if (RB_TYPE_P(hash_or_sym, T_HASH)) {
hash = hash_or_sym;
}
else if (SYMBOL_P(hash_or_sym)) {
key = hash_or_sym;
}
else {
rb_bug("non-hash or symbol given");
}
struct gc_process_stat_total total;
uint64_t direct_global_gc_count;
unsigned int lev = RB_GC_VM_LOCK();
total = global_objspace->process_stat_archive;
rb_gc_vm_each_objspace(gc_process_stat_accumulate_i, &total);
direct_global_gc_count = global_objspace->global_gc.count;
RB_GC_VM_UNLOCK(lev);
rb_objspace_t *const current = rb_gc_get_objspace();
if (!gc_during_gc_get(current)) {
GC_ASSERT(total.global_gc_invocation_count == direct_global_gc_count);
}
/* Convert to Ruby values after all collector locks are released. */
uint64_t time_ns = total.marking_time_ns + total.sweeping_time_ns;
uint64_t count = total.minor_gc_count + total.major_gc_count + total.global_gc_invocation_count;
size_t page_pool_arenas = global_objspace->page_pool.arena_count;
#define SET64(name, attr) \
if (key == gc_stat_symbols[gc_stat_sym_##name]) \
return ULL2NUM(attr); \
else if (hash != Qnil) \
rb_hash_aset(hash, gc_stat_symbols[gc_stat_sym_##name], ULL2NUM(attr));
SET64(count, count);
SET64(time, ns_to_ms(time_ns));
SET64(marking_time, ns_to_ms(total.marking_time_ns));
SET64(sweeping_time, ns_to_ms(total.sweeping_time_ns));
SET64(minor_gc_count, total.minor_gc_count);
SET64(major_gc_count, total.major_gc_count);
SET64(global_gc_count, total.global_gc_invocation_count);
SET64(page_pool_arenas, page_pool_arenas);
SET64(page_pool_arenas_freed, global_objspace->page_pool.arenas_unmapped);
SET64(page_pool_total_pages, (uint64_t)page_pool_arenas * PAGE_POOL_ARENA_BODIES);
SET64(page_pool_discarded_pages, global_objspace->page_pool.advised_count);
#undef SET64
if (!NIL_P(key)) {
/* Matched key should return above. */
return Qundef;
}
return hash;
}
VALUE
rb_gc_impl_stat(void *objspace_ptr, VALUE hash_or_sym)
{
if (objspace_ptr == NULL) {
return gc_process_stat(hash_or_sym);
}
rb_objspace_t *objspace = objspace_ptr;
VALUE hash = Qnil, key = Qnil;
malloc_increase_local_flush(objspace);
if (RB_TYPE_P(hash_or_sym, T_HASH)) {
hash = hash_or_sym;
}
else if (SYMBOL_P(hash_or_sym)) {
key = hash_or_sym;
}
else {
rb_bug("non-hash or symbol given");
}
#define SET(name, attr) \
if (key == gc_stat_symbols[gc_stat_sym_##name]) \
return SIZET2NUM(attr); \
else if (hash != Qnil) \
rb_hash_aset(hash, gc_stat_symbols[gc_stat_sym_##name], SIZET2NUM(attr));
#define SET64(name, attr) \
if (key == gc_stat_symbols[gc_stat_sym_##name]) \
return ULL2NUM(attr); \
else if (hash != Qnil) \
rb_hash_aset(hash, gc_stat_symbols[gc_stat_sym_##name], ULL2NUM(attr));
SET(count, objspace->profile.count);
SET(time, (size_t)ns_to_ms(objspace->profile.marking_time_ns + objspace->profile.sweeping_time_ns)); // TODO: UINT64T2NUM
SET(marking_time, (size_t)ns_to_ms(objspace->profile.marking_time_ns));
SET(sweeping_time, (size_t)ns_to_ms(objspace->profile.sweeping_time_ns));
{
uint64_t total_malloc = (uint64_t)gc_counter_load_relaxed(&objspace->malloc_counters.counters.malloc);
uint64_t total_free = (uint64_t)gc_counter_load_relaxed(&objspace->malloc_counters.counters.free);
SET64(total_malloc_bytes, total_malloc);
SET64(total_free_bytes, total_free);
}
/* implementation dependent counters (small / fixnum-safe) */
SET(heap_allocated_pages, rb_darray_size(objspace->heap_pages.sorted));
SET(heap_empty_pages, objspace->empty_pages_count)
SET(heap_allocatable_bytes, objspace->heap_pages.allocatable_bytes);
SET(heap_eden_pages, heap_eden_total_pages(objspace));
SET(total_allocated_pages, objspace->heap_pages.allocated_pages);
SET(total_freed_pages, objspace->heap_pages.freed_pages);
SET(malloc_increase_bytes, gc_malloc_counters_increase_unsigned(objspace, &objspace->malloc_counters.counters));
SET(malloc_increase_bytes_limit, malloc_limit);
SET(minor_gc_count, objspace->profile.minor_gc_count);
SET(major_gc_count, objspace->profile.major_gc_count);
SET(global_gc_count, objspace->profile.global_gc_count);
SET(global_gc_invocation_count, objspace->profile.global_gc_invocation_count);
SET(compact_count, objspace->profile.compact_count);
SET(read_barrier_faults, objspace->profile.read_barrier_faults);
SET(total_moved_objects, objspace->rcompactor.total_moved);
SET(remembered_wb_unprotected_objects, objspace->rgengc.uncollectible_wb_unprotected_objects);
SET(remembered_wb_unprotected_objects_limit, objspace->rgengc.uncollectible_wb_unprotected_objects_limit);
SET(old_objects, objspace->rgengc.old_objects);
SET(old_objects_limit, objspace->rgengc.old_objects_limit);
#if RGENGC_ESTIMATE_OLDMALLOC
SET(oldmalloc_increase_bytes, gc_oldmalloc_increase_load(objspace));
SET(oldmalloc_increase_bytes_limit, objspace->rgengc.oldmalloc_increase_limit);
#endif
SET(total_allocated_objects, total_allocated_objects(objspace));
SET(total_freed_objects, total_freed_objects(objspace));
SET(heap_available_slots, objspace_available_slots(objspace));
SET(heap_live_slots, objspace_live_slots(objspace));
SET(heap_free_slots, objspace_free_slots(objspace));
SET(heap_final_slots, total_final_slots_count(objspace));
SET(heap_marked_slots, objspace->marked_slots);
#if RGENGC_PROFILE
SET(total_generated_normal_object_count, objspace->profile.total_generated_normal_object_count);
SET(total_generated_shady_object_count, objspace->profile.total_generated_shady_object_count);
SET(total_shade_operation_count, objspace->profile.total_shade_operation_count);
SET(total_promoted_count, objspace->profile.total_promoted_count);
SET(total_remembered_normal_object_count, objspace->profile.total_remembered_normal_object_count);
SET(total_remembered_shady_object_count, objspace->profile.total_remembered_shady_object_count);
#endif /* RGENGC_PROFILE */
#undef SET
#undef SET64
if (!NIL_P(key)) {
// Matched key should return above
return Qundef;
}
#if defined(RGENGC_PROFILE) && RGENGC_PROFILE >= 2
if (hash != Qnil) {
gc_count_add_each_types(hash, "generated_normal_object_count_types", objspace->profile.generated_normal_object_count_types);
gc_count_add_each_types(hash, "generated_shady_object_count_types", objspace->profile.generated_shady_object_count_types);
gc_count_add_each_types(hash, "shade_operation_count_types", objspace->profile.shade_operation_count_types);
gc_count_add_each_types(hash, "promoted_types", objspace->profile.promoted_types);
gc_count_add_each_types(hash, "remembered_normal_object_count_types", objspace->profile.remembered_normal_object_count_types);
gc_count_add_each_types(hash, "remembered_shady_object_count_types", objspace->profile.remembered_shady_object_count_types);
}
#endif
return hash;
}
enum gc_stat_heap_sym {
gc_stat_heap_sym_slot_size,
gc_stat_heap_sym_heap_live_slots,
gc_stat_heap_sym_heap_free_slots,
gc_stat_heap_sym_heap_final_slots,
gc_stat_heap_sym_heap_eden_pages,
gc_stat_heap_sym_heap_eden_slots,
gc_stat_heap_sym_total_allocated_pages,
gc_stat_heap_sym_force_major_gc_count,
gc_stat_heap_sym_force_incremental_marking_finish_count,
gc_stat_heap_sym_heap_allocatable_slots,
gc_stat_heap_sym_total_allocated_objects,
gc_stat_heap_sym_total_freed_objects,
gc_stat_heap_sym_last
};
static VALUE gc_stat_heap_symbols[gc_stat_heap_sym_last];
static void
setup_gc_stat_heap_symbols(void)
{
#define S(s) gc_stat_heap_symbols[gc_stat_heap_sym_##s] = ID2SYM(rb_intern_const(#s))
S(slot_size);
S(heap_live_slots);
S(heap_free_slots);
S(heap_final_slots);
S(heap_eden_pages);
S(heap_eden_slots);
S(heap_allocatable_slots);
S(total_allocated_pages);
S(force_major_gc_count);
S(force_incremental_marking_finish_count);
S(total_allocated_objects);
S(total_freed_objects);
#undef S
}
static VALUE
stat_one_heap(rb_objspace_t *objspace, rb_heap_t *heap, VALUE hash, VALUE key)
{
#define SET(name, attr) \
if (key == gc_stat_heap_symbols[gc_stat_heap_sym_##name]) \
return SIZET2NUM(attr); \
else if (hash != Qnil) \
rb_hash_aset(hash, gc_stat_heap_symbols[gc_stat_heap_sym_##name], SIZET2NUM(attr));
SET(slot_size, heap->slot_size);
SET(heap_live_slots, heap->total_allocated_objects - heap->total_freed_objects - heap->final_slots_count);
SET(heap_free_slots, heap->total_slots - (heap->total_allocated_objects - heap->total_freed_objects));
SET(heap_final_slots, heap->final_slots_count);
SET(heap_eden_pages, heap->total_pages);
SET(heap_eden_slots, heap->total_slots);
SET(heap_allocatable_slots, objspace->heap_pages.allocatable_bytes / heap->slot_size);
SET(total_allocated_pages, heap->total_allocated_pages);
SET(force_major_gc_count, heap->force_major_gc_count);
SET(force_incremental_marking_finish_count, heap->force_incremental_marking_finish_count);
SET(total_allocated_objects, heap->total_allocated_objects);
SET(total_freed_objects, heap->total_freed_objects);
#undef SET
if (!NIL_P(key)) {
// Matched key should return above
return Qundef;
}
return hash;
}
VALUE
rb_gc_impl_stat_heap(void *objspace_ptr, VALUE heap_name, VALUE hash_or_sym)
{
rb_objspace_t *objspace = objspace_ptr;
if (NIL_P(heap_name)) {
if (!RB_TYPE_P(hash_or_sym, T_HASH)) {
rb_bug("non-hash given");
}
for (int i = 0; i < HEAP_COUNT; i++) {
VALUE hash = rb_hash_aref(hash_or_sym, INT2FIX(i));
if (NIL_P(hash)) {
hash = rb_hash_new();
rb_hash_aset(hash_or_sym, INT2FIX(i), hash);
}
stat_one_heap(objspace, &heaps[i], hash, Qnil);
}
}
else if (FIXNUM_P(heap_name)) {
int heap_idx = FIX2INT(heap_name);
if (heap_idx < 0 || heap_idx >= HEAP_COUNT) {
rb_raise(rb_eArgError, "size pool index out of range");
}
if (SYMBOL_P(hash_or_sym)) {
return stat_one_heap(objspace, &heaps[heap_idx], Qnil, hash_or_sym);
}
else if (RB_TYPE_P(hash_or_sym, T_HASH)) {
return stat_one_heap(objspace, &heaps[heap_idx], hash_or_sym, Qnil);
}
else {
rb_bug("non-hash or symbol given");
}
}
else {
rb_bug("heap_name must be nil or an Integer");
}
return hash_or_sym;
}
/* I could include internal.h for this, but doing so undefines some Array macros
* necessary for initialising objects, and I don't want to include all the array
* headers to get them back
* TODO: Investigate why RARRAY_AREF gets undefined in internal.h
*/
#ifndef RBOOL
#define RBOOL(v) (v ? Qtrue : Qfalse)
#endif
VALUE
rb_gc_impl_config_get(void *objspace_ptr)
{
#define sym(name) ID2SYM(rb_intern_const(name))
rb_objspace_t *objspace = objspace_ptr;
VALUE hash = rb_hash_new();
rb_hash_aset(hash, sym("rgengc_allow_full_mark"), RBOOL(gc_config_full_mark_val));
return hash;
}
static int
gc_config_set_key(VALUE key, VALUE value, VALUE data)
{
rb_objspace_t *objspace = (rb_objspace_t *)data;
if (rb_sym2id(key) == rb_intern("rgengc_allow_full_mark")) {
gc_rest(objspace);
gc_config_full_mark_set(RTEST(value));
}
return ST_CONTINUE;
}
void
rb_gc_impl_config_set(void *objspace_ptr, VALUE hash)
{
rb_objspace_t *objspace = objspace_ptr;
if (!RB_TYPE_P(hash, T_HASH)) {
rb_raise(rb_eArgError, "expected keyword arguments");
}
rb_hash_foreach(hash, gc_config_set_key, (st_data_t)objspace);
}
VALUE
rb_gc_impl_stress_get(void *objspace_ptr)
{
return ruby_gc_stress_mode;
}
void
rb_gc_impl_stress_set(void *objspace_ptr, VALUE flag)
{
global_objspace->gc_stressful = RTEST(flag);
global_objspace->gc_stress_mode = flag;
}
static int
get_envparam_size(const char *name, size_t *default_value, size_t lower_bound)
{
const char *ptr = getenv(name);
ssize_t val;
if (ptr != NULL && *ptr) {
size_t unit = 0;
char *end;
#if SIZEOF_SIZE_T == SIZEOF_LONG_LONG
val = strtoll(ptr, &end, 0);
#else
val = strtol(ptr, &end, 0);
#endif
switch (*end) {
case 'k': case 'K':
unit = 1024;
++end;
break;
case 'm': case 'M':
unit = 1024*1024;
++end;
break;
case 'g': case 'G':
unit = 1024*1024*1024;
++end;
break;
}
while (*end && isspace((unsigned char)*end)) end++;
if (*end) {
if (RTEST(ruby_verbose)) fprintf(stderr, "invalid string for %s: %s\n", name, ptr);
return 0;
}
if (unit > 0) {
if (val < -(ssize_t)(SIZE_MAX / 2 / unit) || (ssize_t)(SIZE_MAX / 2 / unit) < val) {
if (RTEST(ruby_verbose)) fprintf(stderr, "%s=%s is ignored because it overflows\n", name, ptr);
return 0;
}
val *= unit;
}
if (val > 0 && (size_t)val > lower_bound) {
if (RTEST(ruby_verbose)) {
fprintf(stderr, "%s=%"PRIdSIZE" (default value: %"PRIuSIZE")\n", name, val, *default_value);
}
*default_value = (size_t)val;
return 1;
}
else {
if (RTEST(ruby_verbose)) {
fprintf(stderr, "%s=%"PRIdSIZE" (default value: %"PRIuSIZE") is ignored because it must be greater than %"PRIuSIZE".\n",
name, val, *default_value, lower_bound);
}
return 0;
}
}
return 0;
}
static int
get_envparam_double(const char *name, double *default_value, double lower_bound, double upper_bound, int accept_zero)
{
const char *ptr = getenv(name);
double val;
if (ptr != NULL && *ptr) {
char *end;
val = strtod(ptr, &end);
if (!*ptr || *end) {
if (RTEST(ruby_verbose)) fprintf(stderr, "invalid string for %s: %s\n", name, ptr);
return 0;
}
if (accept_zero && val == 0.0) {
goto accept;
}
else if (val <= lower_bound) {
if (RTEST(ruby_verbose)) {
fprintf(stderr, "%s=%f (default value: %f) is ignored because it must be greater than %f.\n",
name, val, *default_value, lower_bound);
}
}
else if (upper_bound != 0.0 && /* ignore upper_bound if it is 0.0 */
val > upper_bound) {
if (RTEST(ruby_verbose)) {
fprintf(stderr, "%s=%f (default value: %f) is ignored because it must be lower than %f.\n",
name, val, *default_value, upper_bound);
}
}
else {
goto accept;
}
}
return 0;
accept:
if (RTEST(ruby_verbose)) fprintf(stderr, "%s=%f (default value: %f)\n", name, val, *default_value);
*default_value = val;
return 1;
}
/*
* GC tuning environment variables
*
* * RUBY_GC_HEAP_FREE_SLOTS
* - Prepare at least this amount of slots after GC.
* - Allocate slots if there are not enough slots.
* * RUBY_GC_HEAP_GROWTH_FACTOR (new from 2.1)
* - Allocate slots by this factor.
* - (next slots number) = (current slots number) * (this factor)
* * RUBY_GC_HEAP_GROWTH_MAX_BYTES (was RUBY_GC_HEAP_GROWTH_MAX_SLOTS)
* - Allocation rate is limited to this number of bytes.
* * RUBY_GC_HEAP_FREE_SLOTS_MIN_RATIO (new from 2.4)
* - Allocate additional pages when the number of free slots is
* lower than the value (total_slots * (this ratio)).
* * RUBY_GC_HEAP_FREE_SLOTS_GOAL_RATIO (new from 2.4)
* - Allocate slots to satisfy this formula:
* free_slots = total_slots * goal_ratio
* - In other words, prepare (total_slots * goal_ratio) free slots.
* - if this value is 0.0, then use RUBY_GC_HEAP_GROWTH_FACTOR directly.
* * RUBY_GC_HEAP_FREE_SLOTS_MAX_RATIO (new from 2.4)
* - Allow to free pages when the number of free slots is
* greater than the value (total_slots * (this ratio)).
* * RUBY_GC_HEAP_OLDOBJECT_LIMIT_FACTOR (new from 2.1.1)
* - Do full GC when the number of old objects is more than R * N
* where R is this factor and
* N is the number of old objects just after last full GC.
*
* * obsolete
* * RUBY_FREE_MIN -> RUBY_GC_HEAP_FREE_SLOTS (from 2.1)
* * RUBY_HEAP_MIN_SLOTS -> RUBY_GC_HEAP_INIT_SLOTS (from 2.1) -> RUBY_GC_HEAP_INIT_BYTES
*
* * RUBY_GC_MALLOC_LIMIT
* * RUBY_GC_MALLOC_LIMIT_MAX (new from 2.1)
* * RUBY_GC_MALLOC_LIMIT_GROWTH_FACTOR (new from 2.1)
*
* * RUBY_GC_OLDMALLOC_LIMIT (new from 2.1)
* * RUBY_GC_OLDMALLOC_LIMIT_MAX (new from 2.1)
* * RUBY_GC_OLDMALLOC_LIMIT_GROWTH_FACTOR (new from 2.1)
*/
void
rb_gc_impl_set_params(void *objspace_ptr)
{
rb_objspace_t *objspace = objspace_ptr;
get_envparam_size("RUBY_GC_HEAP_FREE_SLOTS", &gc_params.heap_free_slots, 0);
get_envparam_size("RUBY_GC_HEAP_INIT_BYTES", &gc_params.heap_init_bytes,
heap_init_bytes_min() - 1);
get_envparam_size("RUBY_GC_RACTOR_HEAP_INIT_BYTES", &gc_params.ractor_heap_init_bytes,
heap_init_bytes_min() - 1);
get_envparam_double("RUBY_GC_HEAP_GROWTH_FACTOR", &gc_params.growth_factor, 1.0, 0.0, FALSE);
get_envparam_size ("RUBY_GC_HEAP_GROWTH_MAX_BYTES", &gc_params.growth_max_bytes, 0);
get_envparam_double("RUBY_GC_HEAP_FREE_SLOTS_MIN_RATIO", &gc_params.heap_free_slots_min_ratio,
0.0, 1.0, FALSE);
get_envparam_double("RUBY_GC_HEAP_FREE_SLOTS_MAX_RATIO", &gc_params.heap_free_slots_max_ratio,
gc_params.heap_free_slots_min_ratio, 1.0, FALSE);
get_envparam_double("RUBY_GC_HEAP_FREE_SLOTS_GOAL_RATIO", &gc_params.heap_free_slots_goal_ratio,
gc_params.heap_free_slots_min_ratio, gc_params.heap_free_slots_max_ratio, TRUE);
get_envparam_double("RUBY_GC_HEAP_OLDOBJECT_LIMIT_FACTOR", &gc_params.oldobject_limit_factor, 0.0, 0.0, TRUE);
get_envparam_double("RUBY_GC_HEAP_REMEMBERED_WB_UNPROTECTED_OBJECTS_LIMIT_RATIO", &gc_params.uncollectible_wb_unprotected_objects_limit_ratio, 0.0, 0.0, TRUE);
if (get_envparam_size("RUBY_GC_MALLOC_LIMIT", &gc_params.malloc_limit_min, 0)) {
malloc_limit = gc_params.malloc_limit_min;
}
get_envparam_size ("RUBY_GC_MALLOC_LIMIT_MAX", &gc_params.malloc_limit_max, 0);
if (!gc_params.malloc_limit_max) { /* ignore max-check if 0 */
gc_params.malloc_limit_max = SIZE_MAX;
}
get_envparam_double("RUBY_GC_MALLOC_LIMIT_GROWTH_FACTOR", &gc_params.malloc_limit_growth_factor, 1.0, 0.0, FALSE);
#if RGENGC_ESTIMATE_OLDMALLOC
if (get_envparam_size("RUBY_GC_OLDMALLOC_LIMIT", &gc_params.oldmalloc_limit_min, 0)) {
objspace->rgengc.oldmalloc_increase_limit = gc_params.oldmalloc_limit_min;
}
get_envparam_size ("RUBY_GC_OLDMALLOC_LIMIT_MAX", &gc_params.oldmalloc_limit_max, 0);
get_envparam_double("RUBY_GC_OLDMALLOC_LIMIT_GROWTH_FACTOR", &gc_params.oldmalloc_limit_growth_factor, 1.0, 0.0, FALSE);
#endif
}
static inline size_t
objspace_malloc_size(rb_objspace_t *objspace, void *ptr, size_t hint)
{
#ifdef HAVE_MALLOC_USABLE_SIZE
if (!hint) {
hint = malloc_usable_size(ptr);
}
#endif
return hint;
}
enum memop_type {
MEMOP_TYPE_MALLOC = 0,
MEMOP_TYPE_FREE,
MEMOP_TYPE_REALLOC
};
static inline void
atomic_sub_nounderflow(size_t *var, size_t sub)
{
if (sub == 0) return;
while (1) {
size_t val = rbimpl_atomic_size_load(var, RBIMPL_ATOMIC_RELAXED);
size_t decrement = val < sub ? val : sub;
if (RUBY_ATOMIC_SIZE_CAS(*var, val, val-decrement) == val) break;
}
}
#define gc_stress_full_mark_after_malloc_p() \
(FIXNUM_P(ruby_gc_stress_mode) && (FIX2LONG(ruby_gc_stress_mode) & (1<<gc_stress_full_mark_after_malloc)))
static void
objspace_malloc_gc_stress(rb_objspace_t *objspace)
{
if (ruby_gc_stressful && ruby_native_thread_p()) {
unsigned int reason = (GPR_FLAG_IMMEDIATE_MARK | GPR_FLAG_IMMEDIATE_SWEEP |
GPR_FLAG_STRESS | GPR_FLAG_MALLOC);
if (gc_stress_full_mark_after_malloc_p()) {
reason |= GPR_FLAG_FULL_MARK;
}
garbage_collect_with_gvl(objspace, reason);
}
}
static void
malloc_increase_commit(rb_objspace_t *objspace, size_t new_size, size_t old_size)
{
if (new_size > old_size) {
size_t delta = new_size - old_size;
MALLOC_COUNTERS_LOCK(objspace);
gc_counter_add(&objspace->malloc_counters.counters.malloc, delta);
#if RGENGC_ESTIMATE_OLDMALLOC
RUBY_ATOMIC_SIZE_ADD(objspace->malloc_counters.oldmalloc_increase, delta);
#endif
MALLOC_COUNTERS_UNLOCK(objspace);
}
else if (old_size > new_size) {
size_t delta = old_size - new_size;
MALLOC_COUNTERS_LOCK(objspace);
gc_counter_add(&objspace->malloc_counters.counters.free, delta);
#if RGENGC_ESTIMATE_OLDMALLOC
if (during_gc) gc_counter_add(&objspace->malloc_counters.freed_in_gc, delta);
atomic_sub_nounderflow(&objspace->malloc_counters.oldmalloc_increase, delta);
#endif
MALLOC_COUNTERS_UNLOCK(objspace);
}
}
#if USE_MALLOC_INCREASE_LOCAL
static void
malloc_increase_local_flush(rb_objspace_t *objspace)
{
int delta = malloc_increase_local;
if (delta == 0) return;
malloc_increase_local = 0;
if (delta > 0) {
malloc_increase_commit(objspace, (size_t)delta, 0);
}
else {
malloc_increase_commit(objspace, 0, (size_t)(-delta));
}
}
#else
static void
malloc_increase_local_flush(rb_objspace_t *objspace)
{
}
#endif
static inline bool
objspace_malloc_increase_report(rb_objspace_t *objspace, void *mem, size_t new_size, size_t old_size, enum memop_type type, bool gc_allowed)
{
if (0) fprintf(stderr, "increase - ptr: %p, type: %s, new_size: %"PRIdSIZE", old_size: %"PRIdSIZE"\n",
mem,
type == MEMOP_TYPE_MALLOC ? "malloc" :
type == MEMOP_TYPE_FREE ? "free " :
type == MEMOP_TYPE_REALLOC ? "realloc": "error",
new_size, old_size);
return false;
}
static bool
objspace_malloc_increase_body(rb_objspace_t *objspace, void *mem, size_t new_size, size_t old_size, enum memop_type type, bool gc_allowed)
{
#if USE_MALLOC_INCREASE_LOCAL
if (new_size < GC_MALLOC_INCREASE_LOCAL_THRESHOLD &&
old_size < GC_MALLOC_INCREASE_LOCAL_THRESHOLD) {
malloc_increase_local += (int)new_size - (int)old_size;
if (malloc_increase_local >= GC_MALLOC_INCREASE_LOCAL_THRESHOLD ||
malloc_increase_local <= -GC_MALLOC_INCREASE_LOCAL_THRESHOLD) {
malloc_increase_local_flush(objspace);
}
}
else {
malloc_increase_local_flush(objspace);
malloc_increase_commit(objspace, new_size, old_size);
}
#else
malloc_increase_commit(objspace, new_size, old_size);
#endif
if (type == MEMOP_TYPE_MALLOC && gc_allowed) {
retry:
if (malloc_increase > malloc_limit && ruby_native_thread_p() && !dont_gc_val() && !rb_gc_gc_disabled_global_p()) {
if (ruby_thread_has_gvl_p() && is_lazy_sweeping(objspace)) {
gc_sweep_step_for_malloc(objspace); /* sweeping frees may reduce malloc_increase */
goto retry;
}
garbage_collect_with_gvl(objspace, GPR_FLAG_MALLOC);
}
}
#if MALLOC_ALLOCATED_SIZE
if (new_size >= old_size) {
RUBY_ATOMIC_SIZE_ADD(objspace->malloc_params.allocated_size, new_size - old_size);
}
else {
size_t dec_size = old_size - new_size;
#if MALLOC_ALLOCATED_SIZE_CHECK
size_t allocated_size = objspace->malloc_params.allocated_size;
if (allocated_size < dec_size) {
rb_bug("objspace_malloc_increase: underflow malloc_params.allocated_size.");
}
#endif
atomic_sub_nounderflow(&objspace->malloc_params.allocated_size, dec_size);
}
switch (type) {
case MEMOP_TYPE_MALLOC:
RUBY_ATOMIC_SIZE_INC(objspace->malloc_params.allocations);
break;
case MEMOP_TYPE_FREE:
{
size_t allocations = objspace->malloc_params.allocations;
if (allocations > 0) {
atomic_sub_nounderflow(&objspace->malloc_params.allocations, 1);
}
#if MALLOC_ALLOCATED_SIZE_CHECK
else {
GC_ASSERT(objspace->malloc_params.allocations > 0);
}
#endif
}
break;
case MEMOP_TYPE_REALLOC: /* ignore */ break;
}
#endif
return true;
}
#define objspace_malloc_increase(...) \
for (bool malloc_increase_done = objspace_malloc_increase_report(__VA_ARGS__); \
!malloc_increase_done; \
malloc_increase_done = objspace_malloc_increase_body(__VA_ARGS__))
struct malloc_obj_info { /* 4 words */
size_t size;
};
static inline size_t
objspace_malloc_prepare(rb_objspace_t *objspace, size_t size)
{
if (size == 0) size = 1;
#if CALC_EXACT_MALLOC_SIZE
size += sizeof(struct malloc_obj_info);
#endif
return size;
}
static bool
malloc_during_gc_p(rb_objspace_t *objspace)
{
/* malloc is not allowed during GC when we're not using multiple ractors
* (since ractors can run while another thread is sweeping) and when we
* have the GVL (since if we don't have the GVL, we'll try to acquire the
* GVL which will block and ensure the other thread finishes GC). */
return during_gc && !dont_gc_val() && !rb_gc_multi_ractor_p() && ruby_thread_has_gvl_p();
}
static inline void *
objspace_malloc_fixup(rb_objspace_t *objspace, void *mem, size_t size, bool gc_allowed)
{
size = objspace_malloc_size(objspace, mem, size);
objspace_malloc_increase(objspace, mem, size, 0, MEMOP_TYPE_MALLOC, gc_allowed) {}
#if CALC_EXACT_MALLOC_SIZE
{
struct malloc_obj_info *info = mem;
info->size = size;
mem = info + 1;
}
#endif
return mem;
}
#if defined(__GNUC__) && RUBY_DEBUG
#define RB_BUG_INSTEAD_OF_RB_MEMERROR 1
#endif
#ifndef RB_BUG_INSTEAD_OF_RB_MEMERROR
# define RB_BUG_INSTEAD_OF_RB_MEMERROR 0
#endif
#define GC_MEMERROR(...) \
((RB_BUG_INSTEAD_OF_RB_MEMERROR+0) ? rb_bug("" __VA_ARGS__) : (void)0)
#define TRY_WITH_GC(siz, expr) do { \
const gc_profile_record_flag gpr = \
GPR_FLAG_FULL_MARK | \
GPR_FLAG_IMMEDIATE_MARK | \
GPR_FLAG_IMMEDIATE_SWEEP | \
GPR_FLAG_MALLOC; \
/* stress GC must also honor gc_allowed (malloc_gc_disabled) */ \
if (gc_allowed) objspace_malloc_gc_stress(objspace); \
\
if (RB_LIKELY((expr))) { \
/* Success on 1st try */ \
} \
else if (gc_allowed && !garbage_collect_with_gvl(objspace, gpr)) { \
/* @shyouhei thinks this doesn't happen */ \
GC_MEMERROR("TRY_WITH_GC: could not GC"); \
} \
else if ((expr)) { \
/* Success on 2nd try */ \
} \
else { \
GC_MEMERROR("TRY_WITH_GC: could not allocate:" \
"%"PRIdSIZE" bytes for %s", \
siz, # expr); \
} \
} while (0)
static void
check_malloc_not_in_gc(rb_objspace_t *objspace, const char *msg)
{
if (RB_UNLIKELY(malloc_during_gc_p(objspace))) {
dont_gc_on();
during_gc = false;
rb_bug("Cannot %s during GC", msg);
}
}
void
rb_gc_impl_free(void *objspace_ptr, void *ptr, size_t old_size)
{
rb_objspace_t *objspace = objspace_ptr;
if (!ptr) {
/*
* ISO/IEC 9899 says "If ptr is a null pointer, no action occurs" since
* its first version. We would better follow.
*/
return;
}
#if CALC_EXACT_MALLOC_SIZE
struct malloc_obj_info *info = (struct malloc_obj_info *)ptr - 1;
#if VERIFY_FREE_SIZE
if (!info->size) {
const char *freeing = gc_freeing_obj_info();
rb_bug("buffer %p has no recorded size%s%s. Was it allocated with ruby_mimalloc? If so it should be freed with ruby_mimfree", ptr,
freeing ? ", while freeing " : "", freeing ? freeing : "");
}
if (old_size && (old_size + sizeof(struct malloc_obj_info)) != info->size) {
const char *freeing = gc_freeing_obj_info();
rb_bug("buffer %p freed with old_size=%zu, but was allocated with size=%zu%s%s", ptr, old_size, info->size - sizeof(struct malloc_obj_info),
freeing ? ", while freeing " : "", freeing ? freeing : "");
}
#endif
ptr = info;
old_size = info->size;
#endif
old_size = objspace_malloc_size(objspace, ptr, old_size);
objspace_malloc_increase(objspace, ptr, 0, old_size, MEMOP_TYPE_FREE, true) {
free(ptr);
ptr = NULL;
RB_DEBUG_COUNTER_INC(heap_xfree);
}
}
void *
rb_gc_impl_malloc(void *objspace_ptr, size_t size, bool gc_allowed)
{
rb_objspace_t *objspace = objspace_ptr;
check_malloc_not_in_gc(objspace, "malloc");
void *mem;
size = objspace_malloc_prepare(objspace, size);
TRY_WITH_GC(size, mem = malloc(size));
RB_DEBUG_COUNTER_INC(heap_xmalloc);
if (!mem) return mem;
return objspace_malloc_fixup(objspace, mem, size, gc_allowed);
}
void *
rb_gc_impl_calloc(void *objspace_ptr, size_t size, bool gc_allowed)
{
rb_objspace_t *objspace = objspace_ptr;
if (RB_UNLIKELY(malloc_during_gc_p(objspace))) {
rb_warn("calloc during GC detected, this could cause crashes if it triggers another GC");
#if RGENGC_CHECK_MODE || RUBY_DEBUG
rb_bug("Cannot calloc during GC");
#endif
}
void *mem;
size = objspace_malloc_prepare(objspace, size);
TRY_WITH_GC(size, mem = calloc1(size));
if (!mem) return mem;
return objspace_malloc_fixup(objspace, mem, size, gc_allowed);
}
void *
rb_gc_impl_realloc(void *objspace_ptr, void *ptr, size_t new_size, size_t old_size, bool gc_allowed)
{
rb_objspace_t *objspace = objspace_ptr;
check_malloc_not_in_gc(objspace, "realloc");
void *mem;
if (!ptr) return rb_gc_impl_malloc(objspace, new_size, gc_allowed);
/*
* The behavior of realloc(ptr, 0) is implementation defined.
* Therefore we don't use realloc(ptr, 0) for portability reason.
* see http://www.open-std.org/jtc1/sc22/wg14/www/docs/dr_400.htm
*/
if (new_size == 0) {
if ((mem = rb_gc_impl_malloc(objspace, 0, gc_allowed)) != NULL) {
/*
* - OpenBSD's malloc(3) man page says that when 0 is passed, it
* returns a non-NULL pointer to an access-protected memory page.
* The returned pointer cannot be read / written at all, but
* still be a valid argument of free().
*
* https://man.openbsd.org/malloc.3
*
* - Linux's malloc(3) man page says that it _might_ perhaps return
* a non-NULL pointer when its argument is 0. That return value
* is safe (and is expected) to be passed to free().
*
* https://man7.org/linux/man-pages/man3/malloc.3.html
*
* - As I read the implementation jemalloc's malloc() returns fully
* normal 16 bytes memory region when its argument is 0.
*
* - As I read the implementation musl libc's malloc() returns
* fully normal 32 bytes memory region when its argument is 0.
*
* - Other malloc implementations can also return non-NULL.
*/
rb_gc_impl_free(objspace, ptr, old_size);
return mem;
}
else {
/*
* It is dangerous to return NULL here, because that could lead to
* RCE. Fallback to 1 byte instead of zero.
*
* https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-11932
*/
new_size = 1;
}
}
#if CALC_EXACT_MALLOC_SIZE
{
struct malloc_obj_info *info = (struct malloc_obj_info *)ptr - 1;
new_size += sizeof(struct malloc_obj_info);
ptr = info;
#if VERIFY_FREE_SIZE
if (old_size && (old_size + sizeof(struct malloc_obj_info)) != info->size) {
const char *freeing = gc_freeing_obj_info();
rb_bug("buffer %p realloced with old_size=%zu, but was allocated with size=%zu%s%s", ptr, old_size, info->size - sizeof(struct malloc_obj_info),
freeing ? ", while freeing " : "", freeing ? freeing : "");
}
#endif
old_size = info->size;
}
#endif
old_size = objspace_malloc_size(objspace, ptr, old_size);
TRY_WITH_GC(new_size, mem = RB_GNUC_EXTENSION_BLOCK(realloc(ptr, new_size)));
if (!mem) return mem;
new_size = objspace_malloc_size(objspace, mem, new_size);
#if CALC_EXACT_MALLOC_SIZE
{
struct malloc_obj_info *info = mem;
info->size = new_size;
mem = info + 1;
}
#endif
objspace_malloc_increase(objspace, mem, new_size, old_size, MEMOP_TYPE_REALLOC, gc_allowed);
RB_DEBUG_COUNTER_INC(heap_xrealloc);
return mem;
}
void
rb_gc_impl_adjust_memory_usage(void *objspace_ptr, ssize_t diff)
{
rb_objspace_t *objspace = objspace_ptr;
if (diff > 0) {
objspace_malloc_increase(objspace, 0, diff, 0, MEMOP_TYPE_REALLOC, true);
}
else if (diff < 0) {
objspace_malloc_increase(objspace, 0, 0, -diff, MEMOP_TYPE_REALLOC, true);
}
}
// TODO: move GC profiler stuff back into gc.c
/*
------------------------------ GC profiler ------------------------------
*/
#define GC_PROFILE_RECORD_DEFAULT_SIZE 100
#define GC_PROFILE_RECORD_DEFAULT_MAX_RECORDS 4096
#define GC_PROFILE_RECORD_UNBOUNDED 0
static bool
current_process_time(struct timespec *ts)
{
#if defined(HAVE_CLOCK_GETTIME) && defined(CLOCK_PROCESS_CPUTIME_ID)
{
static int try_clock_gettime = 1;
if (try_clock_gettime) {
if (clock_gettime(CLOCK_PROCESS_CPUTIME_ID, ts) == 0) {
return true;
}
else {
try_clock_gettime = 0;
}
}
}
#endif
#ifdef RUSAGE_SELF
{
struct rusage usage;
struct timeval time;
if (getrusage(RUSAGE_SELF, &usage) == 0) {
time = usage.ru_utime;
ts->tv_sec = time.tv_sec;
ts->tv_nsec = (int32_t)time.tv_usec * 1000;
return true;
}
}
#endif
#ifdef _WIN32
{
FILETIME creation_time, exit_time, kernel_time, user_time;
ULARGE_INTEGER ui;
if (GetProcessTimes(GetCurrentProcess(),
&creation_time, &exit_time, &kernel_time, &user_time) != 0) {
memcpy(&ui, &user_time, sizeof(FILETIME));
#define PER100NSEC (uint64_t)(1000 * 1000 * 10)
ts->tv_nsec = (long)(ui.QuadPart % PER100NSEC);
ts->tv_sec = (time_t)(ui.QuadPart / PER100NSEC);
return true;
}
}
#endif
return false;
}
static double
getrusage_time(void)
{
struct timespec ts;
if (current_process_time(&ts)) {
return ts.tv_sec + ts.tv_nsec * 1e-9;
}
else {
return 0.0;
}
}
static inline double
hrtime_to_sec(rb_hrtime_t time)
{
return (double)time / (double)RB_HRTIME_PER_SEC;
}
static inline rb_hrtime_t
elapsed_hrtime_from(rb_hrtime_t start)
{
return rb_hrtime_sub(rb_hrtime_now(), start);
}
static inline size_t
gc_profile_record_count(rb_objspace_t *objspace)
{
return objspace->profile.record_count;
}
static inline size_t
gc_profile_record_index(rb_objspace_t *objspace, size_t logical_index)
{
if (objspace->profile.max_records != GC_PROFILE_RECORD_UNBOUNDED &&
objspace->profile.record_count == objspace->profile.size) {
return (objspace->profile.next_index + logical_index) % objspace->profile.size;
}
else {
return logical_index;
}
}
static void
gc_profile_records_free(rb_objspace_t *objspace)
{
void *p = objspace->profile.records;
objspace->profile.records = NULL;
objspace->profile.size = 0;
objspace->profile.next_index = 0;
objspace->profile.record_count = 0;
objspace->profile.current_record = 0;
free(p);
}
static inline void
gc_prof_setup_new_record(rb_objspace_t *objspace, unsigned int reason)
{
if (objspace->profile.run) {
size_t index;
gc_profile_record *record;
if (objspace->profile.max_records == GC_PROFILE_RECORD_UNBOUNDED) {
index = objspace->profile.record_count++;
objspace->profile.next_index = objspace->profile.record_count;
if (!objspace->profile.records) {
objspace->profile.size = GC_PROFILE_RECORD_DEFAULT_SIZE;
objspace->profile.records = malloc(xmalloc2_size(sizeof(gc_profile_record), objspace->profile.size));
}
if (index >= objspace->profile.size) {
void *ptr;
objspace->profile.size += 1000;
ptr = realloc(objspace->profile.records, xmalloc2_size(sizeof(gc_profile_record), objspace->profile.size));
if (!ptr) rb_memerror();
objspace->profile.records = ptr;
}
}
else {
if (!objspace->profile.records) {
objspace->profile.size = objspace->profile.max_records;
objspace->profile.records = malloc(xmalloc2_size(sizeof(gc_profile_record), objspace->profile.size));
}
index = objspace->profile.next_index;
objspace->profile.next_index = (objspace->profile.next_index + 1) % objspace->profile.size;
if (objspace->profile.record_count < objspace->profile.size) {
objspace->profile.record_count++;
}
}
if (!objspace->profile.records) {
rb_bug("gc_profile malloc or realloc miss");
}
record = objspace->profile.current_record = &objspace->profile.records[index];
MEMZERO(record, gc_profile_record, 1);
/* setup before-GC parameter */
record->flags = reason | (ruby_gc_stressful ? GPR_FLAG_STRESS : 0);
record->sequence = objspace->profile.record_sequence++;
record->gc_invoke_wall_time = rb_hrtime_sub(rb_hrtime_now(),
objspace->profile.invoke_wall_time);
#if MALLOC_ALLOCATED_SIZE
record->allocated_size = malloc_allocated_size;
#endif
#if GC_PROFILE_MORE_DETAIL && GC_PROFILE_DETAIL_MEMORY
#ifdef RUSAGE_SELF
{
struct rusage usage;
if (getrusage(RUSAGE_SELF, &usage) == 0) {
record->maxrss = usage.ru_maxrss;
record->minflt = usage.ru_minflt;
record->majflt = usage.ru_majflt;
}
}
#endif
#endif
}
}
static inline void
gc_prof_timer_start(rb_objspace_t *objspace)
{
if (gc_prof_enabled(objspace)) {
gc_profile_record *record = gc_prof_record(objspace);
#if GC_PROFILE_MORE_DETAIL
record->prepare_time = objspace->profile.prepare_time;
#endif
record->gc_time = 0;
record->gc_invoke_time = getrusage_time();
objspace->profile.gc_wall_start_time = rb_hrtime_now();
}
}
static double
elapsed_time_from(double time)
{
double now = getrusage_time();
if (now > time) {
return now - time;
}
else {
return 0;
}
}
static inline void
gc_prof_timer_stop(rb_objspace_t *objspace)
{
if (gc_prof_enabled(objspace)) {
gc_profile_record *record = gc_prof_record(objspace);
record->gc_time = elapsed_time_from(record->gc_invoke_time);
record->gc_invoke_time -= objspace->profile.invoke_time;
record->gc_wall_time = elapsed_hrtime_from(objspace->profile.gc_wall_start_time);
}
}
static inline void
gc_prof_mark_timer_start(rb_objspace_t *objspace)
{
RUBY_DTRACE_GC_HOOK(MARK_BEGIN);
#if GC_PROFILE_MORE_DETAIL
if (gc_prof_enabled(objspace)) {
gc_prof_record(objspace)->gc_mark_time = getrusage_time();
}
#endif
}
static inline void
gc_prof_mark_timer_stop(rb_objspace_t *objspace)
{
RUBY_DTRACE_GC_HOOK(MARK_END);
#if GC_PROFILE_MORE_DETAIL
if (gc_prof_enabled(objspace)) {
gc_profile_record *record = gc_prof_record(objspace);
record->gc_mark_time = elapsed_time_from(record->gc_mark_time);
}
#endif
}
static inline void
gc_prof_sweep_timer_start(rb_objspace_t *objspace)
{
RUBY_DTRACE_GC_HOOK(SWEEP_BEGIN);
if (gc_prof_enabled(objspace)) {
gc_profile_record *record = gc_prof_record(objspace);
if (record->gc_time > 0 || GC_PROFILE_MORE_DETAIL) {
objspace->profile.gc_sweep_start_time = getrusage_time();
objspace->profile.gc_sweep_wall_start_time = rb_hrtime_now();
}
}
}
static inline void
gc_prof_sweep_timer_stop(rb_objspace_t *objspace)
{
RUBY_DTRACE_GC_HOOK(SWEEP_END);
if (gc_prof_enabled(objspace)) {
double sweep_time;
gc_profile_record *record = gc_prof_record(objspace);
if (record->gc_time > 0) {
sweep_time = elapsed_time_from(objspace->profile.gc_sweep_start_time);
/* need to accumulate GC time for lazy sweep after gc() */
record->gc_time += sweep_time;
record->gc_wall_time = rb_hrtime_add(record->gc_wall_time,
elapsed_hrtime_from(objspace->profile.gc_sweep_wall_start_time));
}
else if (GC_PROFILE_MORE_DETAIL) {
sweep_time = elapsed_time_from(objspace->profile.gc_sweep_start_time);
}
#if GC_PROFILE_MORE_DETAIL
record->gc_sweep_time += sweep_time;
if (heap_pages_deferred_final) record->flags |= GPR_FLAG_HAVE_FINALIZE;
#endif
if (heap_pages_deferred_final) objspace->profile.latest_gc_info |= GPR_FLAG_HAVE_FINALIZE;
}
}
static inline void
gc_prof_set_malloc_info(rb_objspace_t *objspace)
{
#if GC_PROFILE_MORE_DETAIL
if (gc_prof_enabled(objspace)) {
gc_profile_record *record = gc_prof_record(objspace);
record->allocate_increase = malloc_increase;
record->allocate_limit = malloc_limit;
}
#endif
}
static inline void
gc_prof_set_heap_info(rb_objspace_t *objspace)
{
if (gc_prof_enabled(objspace)) {
gc_profile_record *record = gc_prof_record(objspace);
/* Sum across all size pools since each has a different slot size. */
size_t total = 0;
size_t use_size = 0;
size_t total_size = 0;
for (int i = 0; i < HEAP_COUNT; i++) {
rb_heap_t *heap = &heaps[i];
size_t heap_live = heap->total_allocated_objects - heap->total_freed_objects - heap->final_slots_count;
total += heap->total_slots;
use_size += heap_live * heap->slot_size;
total_size += heap->total_slots * heap->slot_size;
}
#if GC_PROFILE_MORE_DETAIL
size_t live = objspace->profile.total_allocated_objects_at_gc_start - total_freed_objects(objspace);
record->heap_use_pages = objspace->profile.heap_used_at_gc_start;
record->heap_live_objects = live;
record->heap_free_objects = total - live;
#endif
record->heap_total_objects = total;
record->heap_use_size = use_size;
record->heap_total_size = total_size;
}
}
/*
* call-seq:
* GC::Profiler.clear -> nil
*
* Clears the \GC profiler data.
*
*/
static VALUE
gc_profile_clear(VALUE _)
{
rb_objspace_t *objspace = rb_gc_get_objspace();
gc_profile_records_free(objspace);
return Qnil;
}
/*
* call-seq:
* GC::Profiler.configure(max_records: 4096) -> nil
*
* Configures how many raw profile records are retained by
* GC::Profiler.raw_data.
*
* The profiler keeps at most +max_records+ records in a bounded ring buffer.
* When the buffer is full, newer GC records overwrite the oldest retained
* records. The default limit is 4096 records.
*
* Pass +nil+ to restore the historical unbounded behavior:
*
* GC::Profiler.configure(max_records: nil)
*
* Changing +max_records+ clears existing raw profile data. This method does
* not enable or disable the profiler; use GC::Profiler.enable and
* GC::Profiler.disable for that.
*/
static VALUE
gc_profile_configure(int argc, VALUE *argv, VALUE _)
{
static ID keywords[1] = {0};
VALUE options, max_records;
rb_objspace_t *objspace = rb_gc_get_objspace();
if (!keywords[0]) {
keywords[0] = rb_intern("max_records");
}
rb_scan_args_kw(rb_keyword_given_p(), argc, argv, ":", &options);
rb_get_kwargs(options, keywords, 0, 1, &max_records);
if (max_records == Qundef) {
return Qnil;
}
else if (NIL_P(max_records)) {
objspace->profile.max_records = GC_PROFILE_RECORD_UNBOUNDED;
}
else {
long value = NUM2LONG(max_records);
if (value <= 0) {
rb_raise(rb_eArgError, "max_records must be positive or nil");
}
objspace->profile.max_records = (size_t)value;
}
gc_profile_records_free(objspace);
return Qnil;
}
/*
* call-seq:
* GC::Profiler.raw_data(limit: nil, since: nil) -> [Hash, ...]
*
* Returns an Array of retained raw profile data Hashes ordered from earliest
* to latest by +:GC_INVOKE_TIME+. +limit:+ returns at most the newest
* retained records. +since:+ returns records with +:GC_SEQUENCE+ greater
* than the given sequence.
*
* For example:
*
* [
* {
* :GC_TIME=>1.3000000000000858e-05,
* :GC_INVOKE_TIME=>0.010634999999999999,
* :GC_WALL_TIME=>1.4000000000000001e-05,
* :GC_INVOKE_WALL_TIME=>0.010640000000000000,
* :GC_PAUSE_TIME=>1.5000000000000000e-05,
* :GC_STOP_TIME=>1.0000000000000000e-06,
* :GC_STW_TIME=>1.4000000000000001e-05,
* :GC_MARK_WALL_TIME=>9.0000000000000002e-06,
* :GC_SWEEP_WALL_TIME=>5.0000000000000004e-06,
* :GC_COMPACT_WALL_TIME=>0.0000000000000000e+00,
* :HEAP_USE_SIZE=>289640,
* :HEAP_TOTAL_SIZE=>588960,
* :HEAP_TOTAL_OBJECTS=>14724,
* :GC_IS_MARKED=>false
* },
* # ...
* ]
*
* The keys mean:
*
* +:GC_SEQUENCE+::
* Monotonically increasing sequence number for this profiler record.
* +:GC_TIME+::
* CPU time elapsed in seconds for this GC run. This is process CPU time,
* not elapsed wall-clock time.
* +:GC_INVOKE_TIME+::
* CPU time elapsed in seconds from startup to when the GC was invoked.
* +:GC_WALL_TIME+::
* Monotonic wall-clock counterpart to +:GC_TIME+ for this GC record.
* This does not include time spent stopping other ractors before the VM
* enters GC. Use the phase wall-clock fields below for mark, sweep, and
* compaction attribution.
* +:GC_INVOKE_WALL_TIME+::
* Monotonic wall-clock time elapsed in seconds from startup to when the GC
* was invoked.
* +:GC_PAUSE_TIME+::
* Monotonic wall-clock time elapsed in seconds while user execution was
* blocked by this GC entry, including time to stop other ractors. This
* may include time from incremental marking or lazy sweeping continuation
* charged to this record.
* +:GC_STOP_TIME+::
* Monotonic wall-clock time elapsed in seconds stopping other ractors.
* +:GC_STW_TIME+::
* Monotonic wall-clock time elapsed in seconds after other ractors have
* stopped and before the VM exits GC.
* +:GC_MARK_WALL_TIME+::
* Monotonic wall-clock time elapsed in seconds spent marking for this GC
* record, accumulated across incremental marking continuations.
* +:GC_SWEEP_WALL_TIME+::
* Monotonic wall-clock time elapsed in seconds spent sweeping for this GC
* record, accumulated across lazy sweeping continuations. This does not
* include compaction time, which is reported separately as
* +:GC_COMPACT_WALL_TIME+.
* +:GC_COMPACT_WALL_TIME+::
* Monotonic wall-clock time elapsed in seconds spent compacting for this GC
* record, or +0.0+ if this GC did not compact.
* +:HEAP_USE_SIZE+::
* Total bytes of heap used
* +:HEAP_TOTAL_SIZE+::
* Total size of heap in bytes
* +:HEAP_TOTAL_OBJECTS+::
* Total number of objects
* +:GC_IS_MARKED+::
* Returns +true+ if the GC is in mark phase
*
* The wall-clock timing fields relate to each other as follows:
*
* GC_PAUSE_TIME == GC_STOP_TIME + GC_STW_TIME
*
* +:GC_MARK_WALL_TIME+, +:GC_SWEEP_WALL_TIME+, and +:GC_COMPACT_WALL_TIME+
* report separate phase timings and must not be added to +:GC_WALL_TIME+.
*
* +:GC_WALL_TIME+ is the wall-clock counterpart to +:GC_TIME+ and is nested
* inside +:GC_STW_TIME+, so it must not be added to +:GC_STW_TIME+. The difference
* +GC_STW_TIME - GC_WALL_TIME+ is VM overhead inside the stopped interval
* (GC event hooks, bookkeeping, consistency checks, and continuation work).
*
* If ruby was built with +GC_PROFILE_MORE_DETAIL+, you will also have access
* to the following hash keys:
*
* +:GC_MARK_TIME+::
* +:GC_SWEEP_TIME+::
* +:ALLOCATE_INCREASE+::
* +:ALLOCATE_LIMIT+::
* +:HEAP_USE_PAGES+::
* +:HEAP_LIVE_OBJECTS+::
* +:HEAP_FREE_OBJECTS+::
* +:HAVE_FINALIZE+::
*
*/
static VALUE
gc_profile_record_get(int argc, VALUE *argv, VALUE _)
{
static ID keywords[2] = {0};
VALUE prof, options, limit_value, since_value;
VALUE gc_profile = rb_ary_new();
size_t i, count, matching = 0, skip = 0, limit = SIZE_MAX, since = 0;
bool use_since = false;
rb_objspace_t *objspace = rb_gc_get_objspace();
if (!keywords[0]) {
keywords[0] = rb_intern("limit");
keywords[1] = rb_intern("since");
}
rb_scan_args_kw(rb_keyword_given_p(), argc, argv, ":", &options);
VALUE values[2] = {Qundef, Qundef};
rb_get_kwargs(options, keywords, 0, 2, values);
limit_value = values[0];
since_value = values[1];
if (limit_value != Qundef && !NIL_P(limit_value)) {
long value = NUM2LONG(limit_value);
if (value < 0) {
rb_raise(rb_eArgError, "limit must be non-negative");
}
limit = (size_t)value;
}
if (since_value != Qundef && !NIL_P(since_value)) {
long value = NUM2LONG(since_value);
if (value < 0) {
rb_raise(rb_eArgError, "since must be non-negative");
}
since = (size_t)value;
use_since = true;
}
if (!objspace->profile.run) {
return Qnil;
}
count = gc_profile_record_count(objspace);
for (i = 0; i < count; i++) {
gc_profile_record *record = &objspace->profile.records[gc_profile_record_index(objspace, i)];
if (!use_since || record->sequence > since) {
matching++;
}
}
if (limit < matching) {
skip = matching - limit;
}
for (i = 0; i < count; i++) {
gc_profile_record *record = &objspace->profile.records[gc_profile_record_index(objspace, i)];
if (use_since && record->sequence <= since) {
continue;
}
if (skip > 0) {
skip--;
continue;
}
prof = rb_hash_new();
rb_hash_aset(prof, ID2SYM(rb_intern("GC_FLAGS")), gc_info_decode(objspace, rb_hash_new(), record->flags));
rb_hash_aset(prof, ID2SYM(rb_intern("GC_SEQUENCE")), SIZET2NUM(record->sequence));
rb_hash_aset(prof, ID2SYM(rb_intern("GC_TIME")), DBL2NUM(record->gc_time));
rb_hash_aset(prof, ID2SYM(rb_intern("GC_INVOKE_TIME")), DBL2NUM(record->gc_invoke_time));
rb_hash_aset(prof, ID2SYM(rb_intern("GC_WALL_TIME")),
DBL2NUM(hrtime_to_sec(record->gc_wall_time)));
rb_hash_aset(prof, ID2SYM(rb_intern("GC_INVOKE_WALL_TIME")),
DBL2NUM(hrtime_to_sec(record->gc_invoke_wall_time)));
rb_hash_aset(prof, ID2SYM(rb_intern("GC_PAUSE_TIME")),
DBL2NUM(hrtime_to_sec(record->gc_pause_time)));
rb_hash_aset(prof, ID2SYM(rb_intern("GC_STOP_TIME")),
DBL2NUM(hrtime_to_sec(record->gc_stop_time)));
rb_hash_aset(prof, ID2SYM(rb_intern("GC_STW_TIME")),
DBL2NUM(hrtime_to_sec(record->gc_stw_time)));
rb_hash_aset(prof, ID2SYM(rb_intern("GC_MARK_WALL_TIME")),
DBL2NUM(hrtime_to_sec(record->gc_mark_wall_time)));
rb_hash_aset(prof, ID2SYM(rb_intern("GC_SWEEP_WALL_TIME")),
DBL2NUM(hrtime_to_sec(record->gc_sweep_wall_time)));
rb_hash_aset(prof, ID2SYM(rb_intern("GC_COMPACT_WALL_TIME")),
DBL2NUM(hrtime_to_sec(record->gc_compact_wall_time)));
rb_hash_aset(prof, ID2SYM(rb_intern("HEAP_USE_SIZE")), SIZET2NUM(record->heap_use_size));
rb_hash_aset(prof, ID2SYM(rb_intern("HEAP_TOTAL_SIZE")), SIZET2NUM(record->heap_total_size));
rb_hash_aset(prof, ID2SYM(rb_intern("HEAP_TOTAL_OBJECTS")), SIZET2NUM(record->heap_total_objects));
rb_hash_aset(prof, ID2SYM(rb_intern("MOVED_OBJECTS")), SIZET2NUM(record->moved_objects));
rb_hash_aset(prof, ID2SYM(rb_intern("GC_IS_MARKED")), Qtrue);
#if GC_PROFILE_MORE_DETAIL
rb_hash_aset(prof, ID2SYM(rb_intern("GC_MARK_TIME")), DBL2NUM(record->gc_mark_time));
rb_hash_aset(prof, ID2SYM(rb_intern("GC_SWEEP_TIME")), DBL2NUM(record->gc_sweep_time));
rb_hash_aset(prof, ID2SYM(rb_intern("ALLOCATE_INCREASE")), SIZET2NUM(record->allocate_increase));
rb_hash_aset(prof, ID2SYM(rb_intern("ALLOCATE_LIMIT")), SIZET2NUM(record->allocate_limit));
rb_hash_aset(prof, ID2SYM(rb_intern("HEAP_USE_PAGES")), SIZET2NUM(record->heap_use_pages));
rb_hash_aset(prof, ID2SYM(rb_intern("HEAP_LIVE_OBJECTS")), SIZET2NUM(record->heap_live_objects));
rb_hash_aset(prof, ID2SYM(rb_intern("HEAP_FREE_OBJECTS")), SIZET2NUM(record->heap_free_objects));
rb_hash_aset(prof, ID2SYM(rb_intern("REMOVING_OBJECTS")), SIZET2NUM(record->removing_objects));
rb_hash_aset(prof, ID2SYM(rb_intern("EMPTY_OBJECTS")), SIZET2NUM(record->empty_objects));
rb_hash_aset(prof, ID2SYM(rb_intern("HAVE_FINALIZE")), (record->flags & GPR_FLAG_HAVE_FINALIZE) ? Qtrue : Qfalse);
#endif
#if RGENGC_PROFILE > 0
rb_hash_aset(prof, ID2SYM(rb_intern("OLD_OBJECTS")), SIZET2NUM(record->old_objects));
rb_hash_aset(prof, ID2SYM(rb_intern("REMEMBERED_NORMAL_OBJECTS")), SIZET2NUM(record->remembered_normal_objects));
rb_hash_aset(prof, ID2SYM(rb_intern("REMEMBERED_SHADY_OBJECTS")), SIZET2NUM(record->remembered_shady_objects));
#endif
rb_ary_push(gc_profile, prof);
}
return gc_profile;
}
#if GC_PROFILE_MORE_DETAIL
#define MAJOR_REASON_MAX 0x10
static char *
gc_profile_dump_major_reason(unsigned int flags, char *buff)
{
unsigned int reason = flags & GPR_FLAG_MAJOR_MASK;
int i = 0;
if (reason == GPR_FLAG_NONE) {
buff[0] = '-';
buff[1] = 0;
}
else {
#define C(x, s) \
if (reason & GPR_FLAG_MAJOR_BY_##x) { \
buff[i++] = #x[0]; \
if (i >= MAJOR_REASON_MAX) rb_bug("gc_profile_dump_major_reason: overflow"); \
buff[i] = 0; \
}
C(NOFREE, N);
C(OLDGEN, O);
C(SHADY, S);
#if RGENGC_ESTIMATE_OLDMALLOC
C(OLDMALLOC, M);
#endif
#undef C
}
return buff;
}
#endif
static void
gc_profile_dump_on(VALUE out, VALUE (*append)(VALUE, VALUE))
{
rb_objspace_t *objspace = rb_gc_get_objspace();
size_t count = gc_profile_record_count(objspace);
#ifdef MAJOR_REASON_MAX
char reason_str[MAJOR_REASON_MAX];
#endif
if (objspace->profile.run && count /* > 1 */) {
size_t i;
const gc_profile_record *record;
append(out, rb_sprintf("GC %"PRIuSIZE" invokes.\n", objspace->profile.count));
append(out, rb_str_new_cstr("Index Invoke Time(sec) Use Size(byte) Total Size(byte) Total Object GC Time(ms)\n"));
for (i = 0; i < count; i++) {
record = &objspace->profile.records[gc_profile_record_index(objspace, i)];
append(out, rb_sprintf("%5"PRIuSIZE" %19.3f %20"PRIuSIZE" %20"PRIuSIZE" %20"PRIuSIZE" %30.20f\n",
i+1, record->gc_invoke_time, record->heap_use_size,
record->heap_total_size, record->heap_total_objects, record->gc_time*1000));
}
#if GC_PROFILE_MORE_DETAIL
const char *str = "\n\n" \
"More detail.\n" \
"Prepare Time = Previously GC's rest sweep time\n"
"Index Flags Allocate Inc. Allocate Limit"
#if CALC_EXACT_MALLOC_SIZE
" Allocated Size"
#endif
" Use Page Mark Time(ms) Sweep Time(ms) Prepare Time(ms) LivingObj FreeObj RemovedObj EmptyObj"
#if RGENGC_PROFILE
" OldgenObj RemNormObj RemShadObj"
#endif
#if GC_PROFILE_DETAIL_MEMORY
" MaxRSS(KB) MinorFLT MajorFLT"
#endif
"\n";
append(out, rb_str_new_cstr(str));
for (i = 0; i < count; i++) {
record = &objspace->profile.records[gc_profile_record_index(objspace, i)];
append(out, rb_sprintf("%5"PRIuSIZE" %4s/%c/%6s%c %13"PRIuSIZE" %15"PRIuSIZE
#if CALC_EXACT_MALLOC_SIZE
" %15"PRIuSIZE
#endif
" %9"PRIuSIZE" %17.12f %17.12f %17.12f %10"PRIuSIZE" %10"PRIuSIZE" %10"PRIuSIZE" %10"PRIuSIZE
#if RGENGC_PROFILE
"%10"PRIuSIZE" %10"PRIuSIZE" %10"PRIuSIZE
#endif
#if GC_PROFILE_DETAIL_MEMORY
"%11ld %8ld %8ld"
#endif
"\n",
i+1,
gc_profile_dump_major_reason(record->flags, reason_str),
(record->flags & GPR_FLAG_HAVE_FINALIZE) ? 'F' : '.',
(record->flags & GPR_FLAG_NEWOBJ) ? "NEWOBJ" :
(record->flags & GPR_FLAG_MALLOC) ? "MALLOC" :
(record->flags & GPR_FLAG_METHOD) ? "METHOD" :
(record->flags & GPR_FLAG_CAPI) ? "CAPI__" : "??????",
(record->flags & GPR_FLAG_STRESS) ? '!' : ' ',
record->allocate_increase, record->allocate_limit,
#if CALC_EXACT_MALLOC_SIZE
record->allocated_size,
#endif
record->heap_use_pages,
record->gc_mark_time*1000,
record->gc_sweep_time*1000,
record->prepare_time*1000,
record->heap_live_objects,
record->heap_free_objects,
record->removing_objects,
record->empty_objects
#if RGENGC_PROFILE
,
record->old_objects,
record->remembered_normal_objects,
record->remembered_shady_objects
#endif
#if GC_PROFILE_DETAIL_MEMORY
,
record->maxrss / 1024,
record->minflt,
record->majflt
#endif
));
}
#endif
}
}
/*
* call-seq:
* GC::Profiler.result -> String
*
* Returns a profile data report such as:
*
* GC 1 invokes.
* Index Invoke Time(sec) Use Size(byte) Total Size(byte) Total Object GC time(ms)
* 1 0.012 159240 212940 10647 0.00000000000001530000
*/
static VALUE
gc_profile_result(VALUE _)
{
VALUE str = rb_str_buf_new(0);
gc_profile_dump_on(str, rb_str_buf_append);
return str;
}
/*
* call-seq:
* GC::Profiler.report
* GC::Profiler.report(io)
*
* Writes the GC::Profiler.result to <tt>$stdout</tt> or the given IO object.
*
*/
static VALUE
gc_profile_report(int argc, VALUE *argv, VALUE self)
{
VALUE out;
out = (!rb_check_arity(argc, 0, 1) ? rb_stdout : argv[0]);
gc_profile_dump_on(out, rb_io_write);
return Qnil;
}
/*
* call-seq:
* GC::Profiler.total_time -> float
*
* The total time used for garbage collection in seconds
*/
static VALUE
gc_profile_total_time(VALUE self)
{
double time = 0;
rb_objspace_t *objspace = rb_gc_get_objspace();
if (objspace->profile.run && gc_profile_record_count(objspace) > 0) {
size_t i;
size_t count = gc_profile_record_count(objspace);
for (i = 0; i < count; i++) {
time += objspace->profile.records[gc_profile_record_index(objspace, i)].gc_time;
}
}
return DBL2NUM(time);
}
/*
* call-seq:
* GC::Profiler.enabled? -> true or false
*
* The current status of \GC profile mode.
*/
static VALUE
gc_profile_enable_get(VALUE self)
{
rb_objspace_t *objspace = rb_gc_get_objspace();
return objspace->profile.run ? Qtrue : Qfalse;
}
/*
* call-seq:
* GC::Profiler.enable -> nil
*
* Starts the \GC profiler.
*
*/
static VALUE
gc_profile_enable(VALUE _)
{
rb_objspace_t *objspace = rb_gc_get_objspace();
objspace->profile.run = TRUE;
objspace->profile.current_record = 0;
return Qnil;
}
/*
* call-seq:
* GC::Profiler.disable -> nil
*
* Stops the \GC profiler.
*
*/
static VALUE
gc_profile_disable(VALUE _)
{
rb_objspace_t *objspace = rb_gc_get_objspace();
objspace->profile.run = FALSE;
objspace->profile.current_record = 0;
return Qnil;
}
static void
rb_gc_verify_internal_consistency(void)
{
gc_verify_internal_consistency(rb_gc_get_objspace());
}
/*
* call-seq:
* GC.verify_internal_consistency -> nil
*
* Verifies internal consistency of the GC.
* This method should only be used for debugging.
*
* This method is only expected to work on CRuby.
*/
static VALUE
gc_verify_internal_consistency_m(VALUE dummy)
{
rb_gc_verify_internal_consistency();
return Qnil;
}
#if GC_CAN_COMPILE_COMPACTION
/*
* call-seq:
* GC.auto_compact = flag
*
* Updates automatic compaction mode.
*
* When enabled, the compactor will execute on every major collection.
*
* Enabling compaction will degrade performance on major collections.
*/
static VALUE
gc_set_auto_compact(VALUE _, VALUE v)
{
GC_ASSERT(GC_COMPACTION_SUPPORTED);
ruby_enable_autocompact = RTEST(v);
#if RGENGC_CHECK_MODE
ruby_autocompact_compare_func = NULL;
if (SYMBOL_P(v)) {
ID id = RB_SYM2ID(v);
if (id == rb_intern("empty")) {
ruby_autocompact_compare_func = compare_free_slots;
}
}
#endif
return v;
}
#else
# define gc_set_auto_compact rb_f_notimplement
#endif
#if GC_CAN_COMPILE_COMPACTION
/*
* call-seq:
* GC.auto_compact -> true or false
*
* Returns whether or not automatic compaction has been enabled.
*/
static VALUE
gc_get_auto_compact(VALUE _)
{
return ruby_enable_autocompact ? Qtrue : Qfalse;
}
#else
# define gc_get_auto_compact rb_f_notimplement
#endif
#if GC_CAN_COMPILE_COMPACTION
/*
* call-seq:
* GC.latest_compact_info -> hash
*
* Returns information about object moved in the most recent \GC compaction.
*
* The returned +hash+ contains the following keys:
*
* [considered]
* Hash containing the type of the object as the key and the number of
* objects of that type that were considered for movement.
* [moved]
* Hash containing the type of the object as the key and the number of
* objects of that type that were actually moved.
* [moved_up]
* Hash containing the type of the object as the key and the number of
* objects of that type that were increased in size.
* [moved_down]
* Hash containing the type of the object as the key and the number of
* objects of that type that were decreased in size.
*
* Some objects can't be moved (due to pinning) so these numbers can be used to
* calculate compaction efficiency.
*/
static VALUE
gc_compact_stats(VALUE self)
{
rb_objspace_t *objspace = rb_gc_get_objspace();
VALUE h = rb_hash_new();
VALUE considered = rb_hash_new();
VALUE moved = rb_hash_new();
VALUE moved_up = rb_hash_new();
VALUE moved_down = rb_hash_new();
for (size_t i = 0; i < T_MASK; i++) {
if (objspace->rcompactor.considered_count_table[i]) {
rb_hash_aset(considered, type_sym(i), SIZET2NUM(objspace->rcompactor.considered_count_table[i]));
}
if (objspace->rcompactor.moved_count_table[i]) {
rb_hash_aset(moved, type_sym(i), SIZET2NUM(objspace->rcompactor.moved_count_table[i]));
}
if (objspace->rcompactor.moved_up_count_table[i]) {
rb_hash_aset(moved_up, type_sym(i), SIZET2NUM(objspace->rcompactor.moved_up_count_table[i]));
}
if (objspace->rcompactor.moved_down_count_table[i]) {
rb_hash_aset(moved_down, type_sym(i), SIZET2NUM(objspace->rcompactor.moved_down_count_table[i]));
}
}
rb_hash_aset(h, ID2SYM(rb_intern("considered")), considered);
rb_hash_aset(h, ID2SYM(rb_intern("moved")), moved);
rb_hash_aset(h, ID2SYM(rb_intern("moved_up")), moved_up);
rb_hash_aset(h, ID2SYM(rb_intern("moved_down")), moved_down);
return h;
}
#else
# define gc_compact_stats rb_f_notimplement
#endif
#if GC_CAN_COMPILE_COMPACTION
/*
* call-seq:
* GC.compact -> hash
*
* This function compacts objects together in Ruby's heap. It eliminates
* unused space (or fragmentation) in the heap by moving objects in to that
* unused space. If there is more than 1 running Ractor, it runs a global
* GC compaction (all object spaces).
*
* The returned +hash+ contains statistics about the objects that were moved;
* see GC.latest_compact_info.
*
* This method is only expected to work on CRuby.
*
* To test whether \GC compaction is supported, use the idiom:
*
* GC.respond_to?(:compact)
*/
static VALUE
gc_compact(VALUE self)
{
rb_objspace_t *objspace = rb_gc_get_objspace();
int full_marking_p = gc_config_full_mark_val;
gc_config_full_mark_set(TRUE);
/* Run GC with compaction enabled */
rb_gc_impl_start(rb_gc_get_objspace(), true, true, true, true, true);
gc_config_full_mark_set(full_marking_p);
return gc_compact_stats(self);
}
#else
# define gc_compact rb_f_notimplement
#endif
#if GC_CAN_COMPILE_COMPACTION
struct desired_compaction_pages_i_data {
rb_objspace_t *objspace;
size_t required_slots[HEAP_COUNT];
};
static int
desired_compaction_pages_i(struct heap_page *page, void *data)
{
struct desired_compaction_pages_i_data *tdata = data;
rb_objspace_t *objspace = tdata->objspace;
VALUE vstart = (VALUE)page->start;
VALUE vend = vstart + (VALUE)(page->total_slots * page->heap->slot_size);
for (VALUE v = vstart; v != vend; v += page->heap->slot_size) {
asan_unpoisoning_object(v) {
/* skip T_NONEs; they won't be moved */
if (BUILTIN_TYPE(v) != T_NONE) {
rb_heap_t *dest_pool = gc_compact_destination_pool(objspace, page->heap, v);
size_t dest_pool_idx = dest_pool - heaps;
tdata->required_slots[dest_pool_idx]++;
}
}
}
return 0;
}
/* call-seq:
* GC.verify_compaction_references(toward: nil, double_heap: false) -> hash
*
* Verify compaction reference consistency.
*
* This method is implementation specific. During compaction, objects that
* were moved are replaced with T_MOVED objects. No object should have a
* reference to a T_MOVED object after compaction.
*
* This function expands the heap to ensure room to move all objects,
* compacts the heap to make sure everything moves, updates all references,
* then performs a full \GC. If any object contains a reference to a T_MOVED
* object, that object should be pushed on the mark stack, and will
* make a SEGV.
*/
static VALUE
gc_verify_compaction_references(int argc, VALUE* argv, VALUE self)
{
static ID keywords[3] = {0};
if (!keywords[0]) {
keywords[0] = rb_intern("toward");
keywords[1] = rb_intern("double_heap");
keywords[2] = rb_intern("expand_heap");
}
VALUE options;
rb_scan_args_kw(rb_keyword_given_p(), argc, argv, ":", &options);
VALUE arguments[3] = { Qnil, Qfalse, Qfalse };
int kwarg_count = rb_get_kwargs(options, keywords, 0, 3, arguments);
bool toward_empty = kwarg_count > 0 && SYMBOL_P(arguments[0]) && SYM2ID(arguments[0]) == rb_intern("empty");
bool expand_heap = (kwarg_count > 1 && RTEST(arguments[1])) || (kwarg_count > 2 && RTEST(arguments[2]));
rb_objspace_t *objspace = rb_gc_get_objspace();
/* This verification machinery (heap expansion, toward_empty page ordering, the
* moved-reference walk) is built for a single objspace, so with several demote it
* to a plain full GC. Plain GC.compact does compact them via the global GC. */
if (!rb_gc_single_objspace_p()) {
rb_gc_impl_start(objspace, true, true, true, false, false);
return gc_compact_stats(self);
}
/* Clear the heap. */
rb_gc_impl_start(objspace, true, true, true, false, false);
unsigned int lev = RB_GC_VM_LOCK();
{
gc_rest(objspace);
/* if both double_heap and expand_heap are set, expand_heap takes precedence */
if (expand_heap) {
struct desired_compaction_pages_i_data desired_compaction = {
.objspace = objspace,
.required_slots = {0},
};
/* Work out how many objects want to be in each size pool, taking account of moves */
objspace_each_pages(objspace, desired_compaction_pages_i, &desired_compaction, TRUE);
/* Find out which pool has the most pages */
size_t max_existing_pages = 0;
for (int i = 0; i < HEAP_COUNT; i++) {
rb_heap_t *heap = &heaps[i];
max_existing_pages = MAX(max_existing_pages, heap->total_pages);
}
/* Add pages to each size pool so that compaction is guaranteed to move every object */
for (int i = 0; i < HEAP_COUNT; i++) {
rb_heap_t *heap = &heaps[i];
size_t pages_to_add = 0;
/*
* Step 1: Make sure every pool has the same number of pages, by adding empty pages
* to smaller pools. This is required to make sure the compact cursor can advance
* through all of the pools in `gc_sweep_compact` without hitting the "sweep &
* compact cursors met" condition on some pools before fully compacting others
*/
pages_to_add += max_existing_pages - heap->total_pages;
/*
* Step 2: Now add additional free pages to each size pool sufficient to hold all objects
* that want to be in that size pool, whether moved into it or moved within it
*/
objspace->heap_pages.allocatable_bytes = desired_compaction.required_slots[i] * heap->slot_size;
while (objspace->heap_pages.allocatable_bytes > 0) {
heap_page_allocate_and_initialize(objspace, heap);
}
/*
* Step 3: Add two more pages so that the compact & sweep cursors will meet _after_ all objects
* have been moved, and not on the last iteration of the `gc_sweep_compact` loop
*/
pages_to_add += 2;
for (; pages_to_add > 0; pages_to_add--) {
heap_page_allocate_and_initialize_force(objspace, heap);
}
}
}
if (toward_empty) {
objspace->rcompactor.compare_func = compare_free_slots;
}
}
RB_GC_VM_UNLOCK(lev);
rb_gc_impl_start(rb_gc_get_objspace(), true, true, true, true, false);
rb_objspace_reachable_objects_from_root(root_obj_check_moved_i, objspace);
objspace_each_objects(objspace, heap_check_moved_i, objspace, TRUE);
objspace->rcompactor.compare_func = NULL;
return gc_compact_stats(self);
}
#else
# define gc_verify_compaction_references rb_f_notimplement
#endif
void
rb_gc_impl_objspace_free(void *objspace_ptr)
{
rb_objspace_t *objspace = objspace_ptr;
if (is_lazy_sweeping(objspace))
rb_bug("lazy sweeping underway when freeing object space");
free(objspace->profile.records);
objspace->profile.records = NULL;
for (size_t i = 0; i < rb_darray_size(objspace->heap_pages.sorted); i++) {
heap_page_free(objspace, rb_darray_get(objspace->heap_pages.sorted, i));
}
rb_darray_free_without_gc(objspace->heap_pages.sorted);
heap_pages_lomem = 0;
heap_pages_himem = 0;
for (int i = 0; i < HEAP_COUNT; i++) {
rb_heap_t *heap = &heaps[i];
heap->total_pages = 0;
heap->total_slots = 0;
}
free_stack_chunks(&objspace->mark_stack);
mark_stack_free_cache(&objspace->mark_stack);
rb_darray_free_without_gc(objspace->weak_references);
#ifdef MALLOC_COUNTERS_NEED_LOCK
rb_native_mutex_destroy(&objspace->malloc_counters.lock);
#endif
rb_native_mutex_destroy(&objspace->process_stat.lock);
free(objspace);
}
#if MALLOC_ALLOCATED_SIZE
/*
* call-seq:
* GC.malloc_allocated_size -> Integer
*
* Returns the size of memory allocated by malloc().
*
* Only available if ruby was built with +CALC_EXACT_MALLOC_SIZE+.
*/
static VALUE
gc_malloc_allocated_size(VALUE self)
{
rb_objspace_t *objspace = (rb_objspace_t *)rb_gc_get_objspace();
return ULL2NUM(objspace->malloc_params.allocated_size);
}
/*
* call-seq:
* GC.malloc_allocations -> Integer
*
* Returns the number of malloc() allocations.
*
* Only available if ruby was built with +CALC_EXACT_MALLOC_SIZE+.
*/
static VALUE
gc_malloc_allocations(VALUE self)
{
rb_objspace_t *objspace = (rb_objspace_t *)rb_gc_get_objspace();
return ULL2NUM(objspace->malloc_params.allocations);
}
#endif
void
rb_gc_impl_before_fork(void *objspace_ptr)
{
rb_objspace_t *objspace = objspace_ptr;
objspace->fork_vm_lock_lev = RB_GC_VM_LOCK();
rb_gc_vm_barrier();
}
void
rb_gc_impl_after_fork(void *objspace_ptr, rb_pid_t pid)
{
rb_objspace_t *objspace = objspace_ptr;
if (pid == 0) {
rb_gc_vm_each_objspace(gc_process_stat_after_fork_i, NULL);
}
RB_GC_VM_UNLOCK(objspace->fork_vm_lock_lev);
objspace->fork_vm_lock_lev = 0;
if (pid == 0) { /* child process */
heap_alloc_state_clear(objspace);
/* The forking Ractor becomes the child process's main Ractor. */
global_objspace->main_objspace = objspace;
page_pool_lock_initialize(&rb_global_objspace_instance.page_pool.lock);
}
}
VALUE rb_ident_hash_new_capa(rb_len_t size);
#if GC_DEBUG_STRESS_TO_CLASS
/*
* call-seq:
* GC.add_stress_to_class(class[, ...])
*
* Raises NoMemoryError when allocating an instance of the given classes.
*
*/
static VALUE
rb_gcdebug_add_stress_to_class(int argc, VALUE *argv, VALUE self)
{
rb_objspace_t *objspace = rb_gc_get_objspace();
if (!stress_to_class) {
set_stress_to_class(rb_ident_hash_new_capa(argc));
}
for (int i = 0; i < argc; i++) {
VALUE klass = argv[i];
rb_hash_aset(stress_to_class, klass, Qtrue);
}
return self;
}
/*
* call-seq:
* GC.remove_stress_to_class(class[, ...])
*
* No longer raises NoMemoryError when allocating an instance of the
* given classes.
*
*/
static VALUE
rb_gcdebug_remove_stress_to_class(int argc, VALUE *argv, VALUE self)
{
rb_objspace_t *objspace = rb_gc_get_objspace();
if (stress_to_class) {
for (int i = 0; i < argc; ++i) {
rb_hash_delete(stress_to_class, argv[i]);
}
if (rb_hash_size(stress_to_class) == 0) {
stress_to_class = 0;
}
}
return Qnil;
}
#endif
void *
rb_gc_impl_objspace_alloc(void)
{
global_objspace_init();
rb_objspace_t *objspace = calloc1(sizeof(rb_objspace_t));
if (objspace) {
rb_native_mutex_initialize(&objspace->process_stat.lock);
}
return objspace;
}
void
rb_gc_impl_objspace_init(void *objspace_ptr)
{
rb_objspace_t *objspace = objspace_ptr;
gc_config_full_mark_set(TRUE);
malloc_limit = gc_params.malloc_limit_min;
objspace->shareable_objects_limit = SHAREABLE_OBJECTS_LIMIT_MIN;
#ifdef MALLOC_COUNTERS_NEED_LOCK
rb_native_mutex_initialize(&objspace->malloc_counters.lock);
#endif
/* Shared by every objspace. preregister deduplicates on (func, data). */
objspace->finalize_deferred_pjob = rb_postponed_job_preregister(0, gc_finalize_deferred, NULL);
if (objspace->finalize_deferred_pjob == POSTPONED_JOB_HANDLE_INVALID) {
rb_bug("Could not preregister postponed job for GC");
}
gc_tdata_deferred_free_pjob_ensure();
/* A standard RVALUE (RBasic + embedded VALUEs + debug overhead) must fit
* in at least one pool. In debug builds RVALUE_OVERHEAD can push this
* beyond the 48-byte pool into the 64-byte pool, which is fine. */
GC_ASSERT(rb_gc_impl_size_allocatable_p(sizeof(struct RBasic) + sizeof(VALUE[RBIMPL_RVALUE_EMBED_LEN_MAX])));
for (int i = 0; i < HEAP_COUNT; i++) {
rb_heap_t *heap = &heaps[i];
heap->slot_size = pool_slot_sizes[i];
ccan_list_head_init(&heap->pages);
}
if (global_objspace->main_objspace == NULL) {
/* Single-threaded at boot and the first objspace is main's: compute process-wide
* constants once here. A later objspace_init rewriting them, even with equal
* values, would race other threads' lock-free reads. */
global_objspace->main_objspace = objspace;
init_size_to_heap_idx();
#if defined(INIT_HEAP_PAGE_ALLOC_USE_MMAP)
/* Need to determine if we can use mmap at runtime. */
heap_page_alloc_use_mmap = INIT_HEAP_PAGE_ALLOC_USE_MMAP;
#endif
gc_params.heap_init_bytes = GC_HEAP_INIT_BYTES;
gc_params.ractor_heap_init_bytes = GC_RACTOR_HEAP_INIT_BYTES ? GC_RACTOR_HEAP_INIT_BYTES
: heap_init_bytes_min();
}
// GC.measure_total_time= sets the caller's objspace only; a new Ractor's follows
// its creator's, which is the objspace running this init (main starts it on).
objspace->flags.measure_gc = global_objspace->main_objspace == objspace ? true
: ((rb_objspace_t *)rb_gc_get_objspace())->flags.measure_gc;
rb_darray_make_without_gc(&objspace->heap_pages.sorted, 0);
rb_darray_make_without_gc(&objspace->weak_references, 0);
#if RGENGC_ESTIMATE_OLDMALLOC
objspace->rgengc.oldmalloc_increase_limit = gc_params.oldmalloc_limit_min;
#endif
init_mark_stack(&objspace->mark_stack);
objspace->profile.invoke_time = getrusage_time();
objspace->profile.invoke_wall_time = rb_hrtime_now();
objspace->profile.max_records = GC_PROFILE_RECORD_DEFAULT_MAX_RECORDS;
finalizer_table = st_init_numtable();
gc_process_stat_publish(objspace);
}
void
rb_gc_impl_init(void)
{
/* Fill the symbol tables here, where no other ractor exists yet: they used to
* be filled on first use, guarded by their own first element, so a second
* ractor could see a half-filled table and GC.stat raised on the rest. */
setup_gc_stat_symbols();
setup_gc_stat_heap_symbols();
setup_gc_latest_gc_info_symbols();
VALUE gc_constants = rb_hash_new();
rb_hash_aset(gc_constants, ID2SYM(rb_intern("DEBUG")), GC_DEBUG ? Qtrue : Qfalse);
/* Minimum slot size that fits a standard RVALUE */
size_t rvalue_pool = 0;
for (size_t i = 0; i < HEAP_COUNT; i++) {
if (pool_slot_sizes[i] >= RVALUE_SLOT_SIZE) { rvalue_pool = pool_slot_sizes[i]; break; }
}
rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_SIZE")), SIZET2NUM(rvalue_pool - RVALUE_OVERHEAD));
rb_hash_aset(gc_constants, ID2SYM(rb_intern("RBASIC_SIZE")), SIZET2NUM(sizeof(struct RBasic)));
rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_OVERHEAD")), SIZET2NUM(RVALUE_OVERHEAD));
rb_hash_aset(gc_constants, ID2SYM(rb_intern("HEAP_PAGE_BITMAP_SIZE")), SIZET2NUM(HEAP_PAGE_BITMAP_SIZE));
rb_hash_aset(gc_constants, ID2SYM(rb_intern("HEAP_PAGE_SIZE")), SIZET2NUM(HEAP_PAGE_SIZE));
rb_hash_aset(gc_constants, ID2SYM(rb_intern("HEAP_COUNT")), LONG2FIX(HEAP_COUNT));
rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVARGC_MAX_ALLOCATE_SIZE")), SIZET2NUM(rb_gc_impl_max_allocation_size()));
rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_OLD_AGE")), LONG2FIX(RVALUE_OLD_AGE));
if (RB_BUG_INSTEAD_OF_RB_MEMERROR+0) {
rb_hash_aset(gc_constants, ID2SYM(rb_intern("RB_BUG_INSTEAD_OF_RB_MEMERROR")), Qtrue);
}
OBJ_FREEZE(gc_constants);
/* Internal constants in the garbage collector. */
rb_define_const(rb_mGC, "INTERNAL_CONSTANTS", gc_constants);
if (GC_COMPACTION_SUPPORTED) {
rb_define_singleton_method(rb_mGC, "compact", gc_compact, 0);
rb_define_singleton_method(rb_mGC, "auto_compact", gc_get_auto_compact, 0);
rb_define_singleton_method(rb_mGC, "auto_compact=", gc_set_auto_compact, 1);
rb_define_singleton_method(rb_mGC, "latest_compact_info", gc_compact_stats, 0);
rb_define_singleton_method(rb_mGC, "verify_compaction_references", gc_verify_compaction_references, -1);
}
else {
rb_define_singleton_method(rb_mGC, "compact", rb_f_notimplement, 0);
rb_define_singleton_method(rb_mGC, "auto_compact", rb_f_notimplement, 0);
rb_define_singleton_method(rb_mGC, "auto_compact=", rb_f_notimplement, 1);
rb_define_singleton_method(rb_mGC, "latest_compact_info", rb_f_notimplement, 0);
rb_define_singleton_method(rb_mGC, "verify_compaction_references", rb_f_notimplement, -1);
}
#if GC_DEBUG_STRESS_TO_CLASS
rb_define_singleton_method(rb_mGC, "add_stress_to_class", rb_gcdebug_add_stress_to_class, -1);
rb_define_singleton_method(rb_mGC, "remove_stress_to_class", rb_gcdebug_remove_stress_to_class, -1);
#endif
/* internal methods */
rb_define_singleton_method(rb_mGC, "verify_internal_consistency", gc_verify_internal_consistency_m, 0);
#if MALLOC_ALLOCATED_SIZE
rb_define_singleton_method(rb_mGC, "malloc_allocated_size", gc_malloc_allocated_size, 0);
rb_define_singleton_method(rb_mGC, "malloc_allocations", gc_malloc_allocations, 0);
#endif
/* Document-class: GC::Profiler
*
* The GC profiler provides access to information on GC runs including time,
* length and object space size.
*
* Example:
*
* GC::Profiler.enable
*
* require 'rdoc/rdoc'
*
* GC::Profiler.report
*
* pp GC::Profiler.raw_data
*
* GC::Profiler.disable
*
* GC::Profiler.raw_data returns one Hash per GC run, including CPU time
* fields such as +:GC_TIME+ and wall-clock fields such as +:GC_WALL_TIME+,
* +:GC_PAUSE_TIME+, +:GC_STOP_TIME+, and +:GC_STW_TIME+. +:GC_WALL_TIME+
* is the wall-clock counterpart to +:GC_TIME+, while +:GC_PAUSE_TIME+
* measures how long user execution was blocked by the GC entry.
*
* See also GC.count, GC.malloc_allocated_size and GC.malloc_allocations
*/
VALUE rb_mProfiler = rb_define_module_under(rb_mGC, "Profiler");
rb_define_singleton_method(rb_mProfiler, "enabled?", gc_profile_enable_get, 0);
rb_define_singleton_method(rb_mProfiler, "enable", gc_profile_enable, 0);
rb_define_singleton_method(rb_mProfiler, "raw_data", gc_profile_record_get, -1);
rb_define_singleton_method(rb_mProfiler, "disable", gc_profile_disable, 0);
rb_define_singleton_method(rb_mProfiler, "clear", gc_profile_clear, 0);
rb_define_singleton_method(rb_mProfiler, "configure", gc_profile_configure, -1);
rb_define_singleton_method(rb_mProfiler, "result", gc_profile_result, 0);
rb_define_singleton_method(rb_mProfiler, "report", gc_profile_report, -1);
rb_define_singleton_method(rb_mProfiler, "total_time", gc_profile_total_time, 0);
{
VALUE opts;
/* \GC build options */
rb_define_const(rb_mGC, "OPTS", opts = rb_ary_new());
#define OPT(o) if (o) rb_ary_push(opts, rb_interned_str(#o, sizeof(#o) - 1))
OPT(GC_DEBUG);
OPT(USE_RGENGC);
OPT(RGENGC_DEBUG);
OPT(RGENGC_CHECK_MODE);
OPT(RGENGC_PROFILE);
OPT(RGENGC_ESTIMATE_OLDMALLOC);
OPT(GC_PROFILE_MORE_DETAIL);
OPT(GC_ENABLE_LAZY_SWEEP);
OPT(CALC_EXACT_MALLOC_SIZE);
OPT(MALLOC_ALLOCATED_SIZE);
OPT(MALLOC_ALLOCATED_SIZE_CHECK);
OPT(GC_PROFILE_DETAIL_MEMORY);
OPT(GC_COMPACTION_SUPPORTED);
#undef OPT
OBJ_FREEZE(opts);
}
}
View Git Blame Copy Permalink